diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java b/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java index 5eb3c0d..65cc8f9 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java @@ -1,5 +1,6 @@ package xyz.zinglizingli.common.config; +import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -8,11 +9,15 @@ import xyz.zinglizingli.common.filter.SearchFilter; @Configuration public class FilterConfig{ + @Value("${pic.save.path}") + private String picSavePath; + @Bean public FilterRegistrationBean filterRegist() { FilterRegistrationBean frBean = new FilterRegistrationBean(); frBean.setFilter(new SearchFilter()); frBean.addUrlPatterns("/*"); + frBean.addInitParameter("picSavePath",picSavePath); return frBean; } diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java b/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java index 6e8057d..449b161 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java @@ -3,6 +3,7 @@ package xyz.zinglizingli.common.filter; import com.fasterxml.jackson.databind.ObjectMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; import org.springframework.http.*; import org.springframework.util.LinkedMultiValueMap; @@ -38,6 +39,9 @@ public class SearchFilter implements Filter { private RestTemplate restTemplate; + private String picSavePath; + + private final String SUANWEI_BOOK_REGEX = ""; private final String SUANWEI_BOOK_HTML_REGEX = "/\\d+_\\d+\\.html"; @@ -48,6 +52,7 @@ public class SearchFilter implements Filter { @Override public void init(FilterConfig filterConfig) throws ServletException { + picSavePath = filterConfig.getInitParameter("picSavePath"); picPostFix = new ArrayList<>(); picPostFix.add("jpg"); picPostFix.add("pcx"); @@ -97,6 +102,19 @@ public class SearchFilter implements Filter { try { + OutputStream out = resp.getOutputStream(); + if(requestURI.contains("/localPic/")){ + InputStream input = new FileInputStream(new File(picSavePath+requestURI)); + byte[] b = new byte[4096]; + for (int n; (n = input.read(b)) != -1;) { + out.write(b, 0, n); + } + input.close(); + out.close(); + return; + + } + if (!requestURL.contains("/manhua/")) { filterChain.doFilter(req, resp); return; @@ -175,7 +193,6 @@ public class SearchFilter implements Filter { String realUrl = "https://images.dmzj.com/" + requestURI.substring(15); ResponseEntity resEntity = restTemplate.exchange(realUrl, HttpMethod.GET, requestEntity, Resource.class); InputStream input = resEntity.getBody().getInputStream(); - OutputStream out = resp.getOutputStream(); byte[] b = new byte[4096]; for (int n; (n = input.read(b)) != -1;) { out.write(b, 0, n); diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java index 51cce0c..2191745 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java @@ -1,11 +1,12 @@ package xyz.zinglizingli.common.schedule; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; -import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; +import org.springframework.core.io.Resource; +import org.springframework.http.*; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; @@ -14,8 +15,13 @@ import xyz.zinglizingli.books.po.BookContent; import xyz.zinglizingli.books.po.BookIndex; import xyz.zinglizingli.books.service.BookService; import xyz.zinglizingli.books.util.ExcutorUtils; +import xyz.zinglizingli.books.util.UUIDUtils; import xyz.zinglizingli.common.utils.RestTemplateUtil; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.OutputStream; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.*; @@ -41,6 +47,13 @@ public class CrawlBooksSchedule { private Byte websiteType; + @Value("${pic.save.type}") + private Byte picSaveType; + + @Value("${pic.save.path}") + private String picSavePath; + + private boolean isExcuting = false; @@ -152,6 +165,30 @@ public class CrawlBooksSchedule { if (picMather.find()) { String picSrc = picMather.group(1); + if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){ + restTemplate = RestTemplateUtil.getInstance("iso-8859-1"); + HttpHeaders headers = new HttpHeaders(); + HttpEntity requestEntity = new HttpEntity<>(null, headers); + ResponseEntity resEntity = restTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + InputStream input = resEntity.getBody().getInputStream(); + picSrc = "/localPic/" + updateTimeStr.substring(0,4)+"/"+updateTimeStr.substring(5,7)+"/"+updateTimeStr.substring(8,10) + + UUIDUtils.getUUID32() + + picSrc.substring(picSrc.lastIndexOf(".")); + File picFile = new File(picSavePath+picSrc); + File parentFile = picFile.getParentFile(); + if(!parentFile.exists()){ + parentFile.mkdirs(); + } + OutputStream out = new FileOutputStream(picFile); + byte[] b = new byte[4096]; + for (int n; (n = input.read(b)) != -1;) { + out.write(b, 0, n); + } + out.close(); + input.close(); + + } + Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)

"); Matcher descMatch = descPatten.matcher(body); if (descMatch.find()) { @@ -490,6 +527,31 @@ public class CrawlBooksSchedule { if (picMather.find()) { String picSrc = picMather.group(1); + if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){ + restTemplate = RestTemplateUtil.getInstance("iso-8859-1"); + HttpHeaders headers = new HttpHeaders(); + headers.add("Referer","https://www.biqudao.com"); + HttpEntity requestEntity = new HttpEntity<>(null, headers); + ResponseEntity resEntity = restTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + InputStream input = resEntity.getBody().getInputStream(); + picSrc = "/localPic/" + updateTimeStr.substring(0,2)+"/"+updateTimeStr.substring(3,5)+"/"+updateTimeStr.substring(6,8) + + UUIDUtils.getUUID32() + + picSrc.substring(picSrc.lastIndexOf(".")); + File picFile = new File(picSavePath+picSrc); + File parentFile = picFile.getParentFile(); + if(!parentFile.exists()){ + parentFile.mkdirs(); + } + OutputStream out = new FileOutputStream(picFile); + byte[] b = new byte[4096]; + for (int n; (n = input.read(b)) != -1;) { + out.write(b, 0, n); + } + out.close(); + input.close(); + + } + Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)

"); Matcher descMatch = descPatten.matcher(body); if (descMatch.find()) { diff --git a/novel-front/src/main/resources/application.yml b/novel-front/src/main/resources/application.yml index 67c58de..d31e0e6 100644 --- a/novel-front/src/main/resources/application.yml +++ b/novel-front/src/main/resources/application.yml @@ -4,8 +4,8 @@ server: spring: datasource: url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai - username: books - password: books + username: root + password: test123456 # url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai # username: root # password: test123456 @@ -71,6 +71,15 @@ crawl: website: type: 2 + + + +pic: + save: + type: 1 #图片保存方式, 1不保存，使用网络图片 ，2本地保存 + path: d:/pic #图片保存路径 + + search: schedule: isRunExcute: 0;