From cdf1d0c0845be908b238e507e60a3e020c2fc84e Mon Sep 17 00:00:00 2001 From: xiongxiaoyang <773861846@qq.com> Date: Fri, 6 Dec 2019 10:13:04 +0800 Subject: [PATCH] =?UTF-8?q?v2.0.3=E5=8F=91=E5=B8=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +++--- novel-front/pom.xml | 2 +- .../zinglizingli/common/filter/SearchFilter.java | 3 ++- .../common/schedule/CrawlBooksSchedule.java | 14 +++++++------- novel-front/src/main/resources/application.yml | 6 +++--- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index fb997ca..d059b58 100644 --- a/README.md +++ b/README.md @@ -161,15 +161,15 @@ novel-admin ：平台后台管理系统源码（独立项目，按需安装） **演示地址2** -[点击前往](https://book.100boot.cn/)（前台，推荐）（未上线漫画专区功能） +[点击前往](https://book.100boot.cn/)（前台） **演示地址3** -[点击前往](http://www.jsy66.vip/)（未上线漫画专区功能） +[点击前往](http://www.jsy66.vip/)（前台） ##### 演示地址4（备用） -[点击前往](http://47.106.243.172:8888)（前台）（已上线漫画专区） +[点击前往](http://47.106.243.172:8888)（前台） [点击前往 ](http://47.106.243.172:8888) （后台） (**后台爬虫程序运行会占用大量服务器资源，试用人数过多，服务器压力大，现暂停演示** ) diff --git a/novel-front/pom.xml b/novel-front/pom.xml index 69c0401..d77fa74 100644 --- a/novel-front/pom.xml +++ b/novel-front/pom.xml @@ -10,7 +10,7 @@ xyz.zinglizingli novel-front - 2.0.2.beta + 2.0.3.beta novel-front 小说精品楼-前台web网站 diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java b/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java index 449b161..0cee3d1 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/filter/SearchFilter.java @@ -102,8 +102,8 @@ public class SearchFilter implements Filter { try { - OutputStream out = resp.getOutputStream(); if(requestURI.contains("/localPic/")){ + OutputStream out = resp.getOutputStream(); InputStream input = new FileInputStream(new File(picSavePath+requestURI)); byte[] b = new byte[4096]; for (int n; (n = input.read(b)) != -1;) { @@ -193,6 +193,7 @@ public class SearchFilter implements Filter { String realUrl = "https://images.dmzj.com/" + requestURI.substring(15); ResponseEntity resEntity = restTemplate.exchange(realUrl, HttpMethod.GET, requestEntity, Resource.class); InputStream input = resEntity.getBody().getInputStream(); + OutputStream out = resp.getOutputStream(); byte[] b = new byte[4096]; for (int n; (n = input.read(b)) != -1;) { out.write(b, 0, n); diff --git a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java index 2191745..ef07c51 100644 --- a/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java +++ b/novel-front/src/main/java/xyz/zinglizingli/common/schedule/CrawlBooksSchedule.java @@ -38,7 +38,9 @@ public class CrawlBooksSchedule { @Autowired private BookService bookService; - RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8"); + RestTemplate utf8RestTemplate = RestTemplateUtil.getInstance("utf-8"); + + RestTemplate isoRestTemplate = RestTemplateUtil.getInstance("iso-8859-1"); @Value("${books.lowestScore}") private Float lowestScore; @@ -166,10 +168,9 @@ public class CrawlBooksSchedule { String picSrc = picMather.group(1); if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){ - restTemplate = RestTemplateUtil.getInstance("iso-8859-1"); HttpHeaders headers = new HttpHeaders(); HttpEntity requestEntity = new HttpEntity<>(null, headers); - ResponseEntity resEntity = restTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + ResponseEntity resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); InputStream input = resEntity.getBody().getInputStream(); picSrc = "/localPic/" + updateTimeStr.substring(0,4)+"/"+updateTimeStr.substring(5,7)+"/"+updateTimeStr.substring(8,10) + UUIDUtils.getUUID32() @@ -528,11 +529,10 @@ public class CrawlBooksSchedule { String picSrc = picMather.group(1); if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){ - restTemplate = RestTemplateUtil.getInstance("iso-8859-1"); HttpHeaders headers = new HttpHeaders(); headers.add("Referer","https://www.biqudao.com"); HttpEntity requestEntity = new HttpEntity<>(null, headers); - ResponseEntity resEntity = restTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); + ResponseEntity resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class); InputStream input = resEntity.getBody().getInputStream(); picSrc = "/localPic/" + updateTimeStr.substring(0,2)+"/"+updateTimeStr.substring(3,5)+"/"+updateTimeStr.substring(6,8) + UUIDUtils.getUUID32() @@ -670,7 +670,7 @@ public class CrawlBooksSchedule { } - private String getByHttpClient(String catBookListUrl) { + private String getByHttpClient(String url) { try { /* HttpClient httpClient = new DefaultHttpClient(); HttpGet getReq = new HttpGet(catBookListUrl); @@ -683,7 +683,7 @@ public class CrawlBooksSchedule { return null; }*/ //经测试restTemplate比httpClient效率高出很多倍，所有选择restTemplate - ResponseEntity forEntity = restTemplate.getForEntity(catBookListUrl, String.class); + ResponseEntity forEntity = utf8RestTemplate.getForEntity(url, String.class); if (forEntity.getStatusCode() == HttpStatus.OK) { return forEntity.getBody(); } else { diff --git a/novel-front/src/main/resources/application.yml b/novel-front/src/main/resources/application.yml index d31e0e6..dd5fe56 100644 --- a/novel-front/src/main/resources/application.yml +++ b/novel-front/src/main/resources/application.yml @@ -4,8 +4,8 @@ server: spring: datasource: url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf-8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai - username: root - password: test123456 + username: books + password: books # url: jdbc:mysql://127.0.0.1:3306/books?useUnicode=true&characterEncoding=utf8&useSSL=false&allowPublicKeyRetrieval=true&serverTimezone=Asia/Shanghai # username: root # password: test123456 @@ -77,7 +77,7 @@ crawl: pic: save: type: 1 #图片保存方式, 1不保存，使用网络图片 ，2本地保存 - path: d:/pic #图片保存路径 + path: /var/pic #图片保存路径 search: