201206030/novel
1
0
Fork 0
mirror of https://github.com/201206030/novel.git synced 2025-04-27 07:30:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

增加网络图片本地化开关，可选择使用网络图片还是本地图片

Browse Source
This commit is contained in:
xiongxiaoyang 2019-12-06 09:11:58 +08:00
parent daab096d9f
commit becbb86179
4 changed files with 98 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
novel-front/src/main/java/xyz/zinglizingli/common/config/FilterConfig.java
View File

@ -1,5 +1,6 @@
package xyz.zinglizingli.common.config;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@ -8,11 +9,15 @@ import xyz.zinglizingli.common.filter.SearchFilter;
@Configuration
public class FilterConfig{
@Value("${pic.save.path}")
private String picSavePath;
@Bean
public FilterRegistrationBean filterRegist() {
FilterRegistrationBean frBean = new FilterRegistrationBean();
frBean.setFilter(new SearchFilter());
frBean.addUrlPatterns("/*");
frBean.addInitParameter("picSavePath",picSavePath);
return frBean;
}

19
novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java
View File

@ -3,6 +3,7 @@ package xyz.zinglizingli.common.filter;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
import org.springframework.util.LinkedMultiValueMap;
@ -38,6 +39,9 @@ public class SearchFilter implements Filter {
private RestTemplate restTemplate;
private String picSavePath;
private final String SUANWEI_BOOK_REGEX = "<a\\s+href=\"/(\\d+_\\d+)/\">";
private final String SUANWEI_BOOK_HTML_REGEX = "/\\d+_\\d+\\.html";
@ -48,6 +52,7 @@ public class SearchFilter implements Filter {
@Override
public void init(FilterConfig filterConfig) throws ServletException {
picSavePath = filterConfig.getInitParameter("picSavePath");
picPostFix = new ArrayList<>();
picPostFix.add("jpg");
picPostFix.add("pcx");
@ -97,6 +102,19 @@ public class SearchFilter implements Filter {
try {
OutputStream out = resp.getOutputStream();
if(requestURI.contains("/localPic/")){
InputStream input = new FileInputStream(new File(picSavePath+requestURI));
byte[] b = new byte[4096];
for (int n; (n = input.read(b)) != -1;) {
out.write(b, 0, n);
}
input.close();
out.close();
return;
}
if (!requestURL.contains("/manhua/")) {
filterChain.doFilter(req, resp);
return;
@ -175,7 +193,6 @@ public class SearchFilter implements Filter {
String realUrl = "https://images.dmzj.com/" + requestURI.substring(15);
ResponseEntity<Resource> resEntity = restTemplate.exchange(realUrl, HttpMethod.GET, requestEntity, Resource.class);
InputStream input = resEntity.getBody().getInputStream();
OutputStream out = resp.getOutputStream();
byte[] b = new byte[4096];
for (int n; (n = input.read(b)) != -1;) {
out.write(b, 0, n);

66
novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java
View File

@ -1,11 +1,12 @@
package xyz.zinglizingli.common.schedule;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.core.io.Resource;
import org.springframework.http.*;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@ -14,8 +15,13 @@ import xyz.zinglizingli.books.po.BookContent;
import xyz.zinglizingli.books.po.BookIndex;
import xyz.zinglizingli.books.service.BookService;
import xyz.zinglizingli.books.util.ExcutorUtils;
import xyz.zinglizingli.books.util.UUIDUtils;
import xyz.zinglizingli.common.utils.RestTemplateUtil;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
@ -41,6 +47,13 @@ public class CrawlBooksSchedule {
private Byte websiteType;
@Value("${pic.save.type}")
private Byte picSaveType;
@Value("${pic.save.path}")
private String picSavePath;
private boolean isExcuting = false;
@ -152,6 +165,30 @@ public class CrawlBooksSchedule {
if (picMather.find()) {
String picSrc = picMather.group(1);
if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){
restTemplate = RestTemplateUtil.getInstance("iso-8859-1");
HttpHeaders headers = new HttpHeaders();
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
ResponseEntity<Resource> resEntity = restTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
InputStream input = resEntity.getBody().getInputStream();
picSrc = "/localPic/" + updateTimeStr.substring(0,4)+"/"+updateTimeStr.substring(5,7)+"/"+updateTimeStr.substring(8,10)
+ UUIDUtils.getUUID32()
+ picSrc.substring(picSrc.lastIndexOf("."));
File picFile = new File(picSavePath+picSrc);
File parentFile = picFile.getParentFile();
if(!parentFile.exists()){
parentFile.mkdirs();
}
OutputStream out = new FileOutputStream(picFile);
byte[] b = new byte[4096];
for (int n; (n = input.read(b)) != -1;) {
out.write(b, 0, n);
}
out.close();
input.close();
}
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
Matcher descMatch = descPatten.matcher(body);
if (descMatch.find()) {
@ -490,6 +527,31 @@ public class CrawlBooksSchedule {
if (picMather.find()) {
String picSrc = picMather.group(1);
if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){
restTemplate = RestTemplateUtil.getInstance("iso-8859-1");
HttpHeaders headers = new HttpHeaders();
headers.add("Referer","https://www.biqudao.com");
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
ResponseEntity<Resource> resEntity = restTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
InputStream input = resEntity.getBody().getInputStream();
picSrc = "/localPic/" + updateTimeStr.substring(0,2)+"/"+updateTimeStr.substring(3,5)+"/"+updateTimeStr.substring(6,8)
+ UUIDUtils.getUUID32()
+ picSrc.substring(picSrc.lastIndexOf("."));
File picFile = new File(picSavePath+picSrc);
File parentFile = picFile.getParentFile();
if(!parentFile.exists()){
parentFile.mkdirs();
}
OutputStream out = new FileOutputStream(picFile);
byte[] b = new byte[4096];
for (int n; (n = input.read(b)) != -1;) {
out.write(b, 0, n);
}
out.close();
input.close();
}
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
Matcher descMatch = descPatten.matcher(body);
if (descMatch.find()) {

13
novel-front/src/main/resources/application.yml
View File

@ -4,8 +4,8 @@ server:
spring:
datasource:
url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
username: books
password: books
username: root
password: test123456
# url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai
# username: root
# password: test123456
@ -71,6 +71,15 @@ crawl:
website:
type: 2
pic:
save:
type: 1 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
path: d:/pic #图片保存路径
search:
schedule:
isRunExcute: 0;