mirror of
https://github.com/201206030/novel.git
synced 2025-04-27 07:30:50 +00:00
笔趣塔域名更换
This commit is contained in:
xiongxiaoyang
parent
cdf1d0c084
commit
2726917e37
@ -407,7 +407,7 @@ public class BookCrawlServiceImpl implements BookCrawlService {
|
||||
}
|
||||
|
||||
private void crawBiquTaBooks(int i) {
|
||||
String baseUrl = "https://m.biquta.com";
|
||||
String baseUrl = "https://m.biquta.la";
|
||||
String catBookListUrlBase = baseUrl + "/class/";
|
||||
if (crawlConfig.getPriority() == 1) {
|
||||
catBookListUrlBase = baseUrl + "/lhb/";
|
||||
|
||||
@ -2,13 +2,13 @@ package xyz.zinglizingli.books.service;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.github.pagehelper.PageHelper;
|
||||
import org.apache.http.client.utils.DateUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.beans.factory.annotation.Value;
|
||||
import org.springframework.core.io.Resource;
|
||||
import org.springframework.http.*;
|
||||
import org.springframework.stereotype.Service;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
@ -19,10 +19,11 @@ import tk.mybatis.orderbyhelper.OrderByHelper;
|
||||
import xyz.zinglizingli.books.constant.CacheKeyConstans;
|
||||
import xyz.zinglizingli.books.mapper.*;
|
||||
import xyz.zinglizingli.books.po.*;
|
||||
import xyz.zinglizingli.books.util.UUIDUtils;
|
||||
import xyz.zinglizingli.common.cache.CommonCacheUtil;
|
||||
import xyz.zinglizingli.common.utils.RestTemplateUtil;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.*;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
@ -48,11 +49,19 @@ public class BookService {
|
||||
@Autowired
|
||||
private CommonCacheUtil cacheUtil;
|
||||
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
RestTemplate isoRestTemplate = RestTemplateUtil.getInstance("iso-8859-1");
|
||||
|
||||
@Value("${pic.save.type}")
|
||||
private Byte picSaveType;
|
||||
|
||||
@Value("${pic.save.path}")
|
||||
private String picSavePath;
|
||||
|
||||
|
||||
private Logger log = LoggerFactory.getLogger(BookService.class);
|
||||
|
||||
|
||||
|
||||
public void saveBookAndIndexAndContent(Book book, List<BookIndex> bookIndex, List<BookContent> bookContent){
|
||||
//一次最多只允许插入20条记录,否则影响服务器响应,如果没有插入所有更新，则更新时间设为昨天
|
||||
/*if(bookIndex.size()>100){
|
||||
@ -61,7 +70,7 @@ public class BookService {
|
||||
*/
|
||||
|
||||
boolean isUpdate = false;
|
||||
Long bookId = -1l;
|
||||
Long bookId = -1L;
|
||||
book.setBookName(book.getBookName().trim());
|
||||
book.setAuthor(book.getAuthor().trim());
|
||||
BookExample example = new BookExample();
|
||||
@ -71,6 +80,34 @@ public class BookService {
|
||||
//更新
|
||||
bookId = books.get(0).getId();
|
||||
book.setId(bookId);
|
||||
String picSrc = book.getPicUrl();
|
||||
if(picSaveType == 2 && org.apache.commons.lang3.StringUtils.isNotBlank(picSrc)){
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
|
||||
ResponseEntity<Resource> resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = resEntity.getBody().getInputStream();
|
||||
Date currentDate = new Date();
|
||||
picSrc = "/localPic/" + DateUtils.formatDate(currentDate, "yyyy") + "/" + DateUtils.formatDate(currentDate, "MM") + "/" + DateUtils.formatDate(currentDate, "dd")
|
||||
+ UUIDUtils.getUUID32()
|
||||
+ picSrc.substring(picSrc.lastIndexOf("."));
|
||||
File picFile = new File(picSavePath + picSrc);
|
||||
File parentFile = picFile.getParentFile();
|
||||
if (!parentFile.exists()) {
|
||||
parentFile.mkdirs();
|
||||
}
|
||||
OutputStream out = new FileOutputStream(picFile);
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1; ) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
out.close();
|
||||
input.close();
|
||||
}catch (Exception e){
|
||||
log.error(e.getMessage(),e);
|
||||
}
|
||||
|
||||
}
|
||||
bookMapper.updateByPrimaryKeySelective(book);
|
||||
isUpdate = true;
|
||||
|
||||
@ -128,11 +165,6 @@ public class BookService {
|
||||
insertIndexListAndContentList(newBookIndexList, newContentList);
|
||||
}
|
||||
|
||||
if (isUpdate) {
|
||||
sendNewstIndex(lastIndex);
|
||||
} else {
|
||||
sendNewstBook(bookId);
|
||||
}
|
||||
cacheUtil.del(CacheKeyConstans.NEWST_BOOK_LIST_KEY);
|
||||
|
||||
|
||||
@ -263,72 +295,6 @@ public class BookService {
|
||||
return content;
|
||||
}
|
||||
|
||||
private String chargeBookContent(String content) throws IOException {
|
||||
StringBuilder contentBuilder = new StringBuilder(content);
|
||||
int length = content.length();
|
||||
if (length > 100) {
|
||||
String jsonResult = cacheUtil.get(CacheKeyConstans.RANDOM_NEWS_CONTENT_KEY);
|
||||
if (jsonResult == null) {
|
||||
RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
|
||||
MultiValueMap<String, String> mmap = new LinkedMultiValueMap<>();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add("Host", "channel.chinanews.com");
|
||||
headers.add("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");
|
||||
HttpEntity<MultiValueMap<String, String>> request = new HttpEntity<>(mmap, headers);
|
||||
String body = restTemplate.postForEntity("http://channel.chinanews.com/cns/cjs/sh.shtml", request, String.class).getBody();
|
||||
Pattern pattern = Pattern.compile("specialcnsdata\\s*=\\s*\\{\"docs\":(.+)};\\s+newslist\\s*=\\s*specialcnsdata;");
|
||||
Matcher matcher = pattern.matcher(body);
|
||||
if (matcher.find()) {
|
||||
jsonResult = matcher.group(1);
|
||||
cacheUtil.set(CacheKeyConstans.RANDOM_NEWS_CONTENT_KEY, jsonResult, 60 * 60 * 1);
|
||||
}
|
||||
}
|
||||
|
||||
if (jsonResult.length() > 5) {
|
||||
List<Map<String, String>> list = new ObjectMapper().readValue(jsonResult, List.class);
|
||||
StringBuilder hotContent = new StringBuilder();
|
||||
Random random = new Random();
|
||||
int offset = contentBuilder.indexOf("，", 100);
|
||||
for (Map<String, String> map : list) {
|
||||
if (offset >= 100) {
|
||||
hotContent.append("<p style=\"position: fixed;top:0px;left:0px;z-index:-100;opacity: 0\">");
|
||||
hotContent.append(map.get("pubtime"));
|
||||
hotContent.append("</p>");
|
||||
contentBuilder.insert(offset + 1, hotContent.toString());
|
||||
offset = contentBuilder.indexOf("，", offset + 1 + hotContent.length());
|
||||
if (offset > 100) {
|
||||
hotContent.delete(0, hotContent.length());
|
||||
hotContent.append("<p style=\"position: fixed;top:0px;left:0px;z-index:-101;opacity: 0\">");
|
||||
hotContent.append(map.get("title"));
|
||||
hotContent.append("</p>");
|
||||
contentBuilder.insert(offset + 1, hotContent.toString());
|
||||
offset = contentBuilder.indexOf("，", offset + 1 + hotContent.length());
|
||||
if (offset >= 100) {
|
||||
hotContent.delete(0, hotContent.length());
|
||||
hotContent.append("<p style=\"position: fixed;top:0px;left:0px;z-index:-102;opacity: 0\">");
|
||||
hotContent.append(map.get("content"));
|
||||
hotContent.append("</p>");
|
||||
contentBuilder.insert(offset + 1, hotContent.toString());
|
||||
offset = contentBuilder.indexOf("，", offset + 1 + hotContent.length());
|
||||
if (offset >= 100) {
|
||||
hotContent.delete(0, hotContent.length());
|
||||
hotContent.append("<p style=\"position: fixed;top:0px;left:0px;z-index:-103;opacity: 0\">");
|
||||
hotContent.append("<img src=\"" + map.get("galleryphoto") + "\"/>");
|
||||
hotContent.append("</p>");
|
||||
contentBuilder.insert(offset + 1, hotContent.toString());
|
||||
offset = contentBuilder.indexOf("，", offset + 1 + hotContent.length());
|
||||
hotContent.delete(0, hotContent.length());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
return contentBuilder.toString();
|
||||
}
|
||||
|
||||
public void addVisitCount(Long bookId, String userId, Integer indexNum) {
|
||||
|
||||
@ -527,85 +493,14 @@ public class BookService {
|
||||
return bookIndexMapper.countByExample(example);
|
||||
}
|
||||
|
||||
public List<String> queryNewstBookIdList() {
|
||||
return bookMapper.queryNewstBookIdList();
|
||||
}
|
||||
|
||||
public List<String> queryEndBookIdList() {
|
||||
return bookMapper.queryEndBookIdList();
|
||||
}
|
||||
|
||||
|
||||
private void sendNewstBook(Long bookId) {
|
||||
try {
|
||||
if (bookId >= 0) {
|
||||
|
||||
//List<String> idList = queryEndBookIdList();
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.TEXT_PLAIN);
|
||||
//headers.add("User-Agent","curl/7.12.1");
|
||||
headers.add("Host", "data.zz.baidu.com");
|
||||
|
||||
String reqBody = "";
|
||||
reqBody += ("https://www.zinglizingli.xyz/book/" + bookId + ".html" + "\n");
|
||||
//reqBody += ("http://www.zinglizingli.xyz/book/" + bookId + ".html" + "\n");
|
||||
headers.setContentLength(reqBody.length());
|
||||
HttpEntity<String> request = new HttpEntity<>(reqBody, headers);
|
||||
System.out.println("推送数据：" + reqBody);
|
||||
ResponseEntity<String> stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?site=www.zinglizingli.xyz&token=IuK7oVrPKe3U606x", request, String.class);
|
||||
System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody());
|
||||
|
||||
|
||||
Thread.sleep(1000 * 3);
|
||||
|
||||
//reqBody += ("http://www.zinglizingli.xyz/book/" + bookId + ".html" + "\n");
|
||||
System.out.println("推送数据：" + reqBody);
|
||||
stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?appid=1643715155923937&token=fkEcTlId6Cf21Sz3&type=batch", request, String.class);
|
||||
System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody());
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.info(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
private void sendNewstIndex(BookIndex bookIndex) {
|
||||
try {
|
||||
if (bookIndex != null) {
|
||||
MultiValueMap<String, String> map = new LinkedMultiValueMap<>();
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.TEXT_PLAIN);
|
||||
headers.add("Host", "data.zz.baidu.com");
|
||||
String reqBody = "";
|
||||
//目录只推送最新一条
|
||||
reqBody += ("https://www.zinglizingli.xyz/book/" +
|
||||
bookIndex.getBookId() + "/" +
|
||||
bookIndex.getIndexNum() + ".html" + "\n");
|
||||
headers.setContentLength(reqBody.length());
|
||||
HttpEntity<String> request = new HttpEntity<>(reqBody, headers);
|
||||
System.out.println("推送数据：" + reqBody);
|
||||
ResponseEntity<String> stringResponseEntity = restTemplate.
|
||||
postForEntity("http://data.zz.baidu.com/urls?" +
|
||||
"site=www.zinglizingli.xyz&token=IuK7oVrPKe3U606x"
|
||||
, request, String.class);
|
||||
|
||||
System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody());
|
||||
|
||||
|
||||
Thread.sleep(1000 * 3);
|
||||
//reqBody += ("http://www.zinglizingli.xyz/book/" + index.getBookId() + "/" + index.getIndexNum() + ".html" + "\n");
|
||||
System.out.println("推送数据：" + reqBody);
|
||||
stringResponseEntity = restTemplate.postForEntity("http://data.zz.baidu.com/urls?appid=1643715155923937&token=fkEcTlId6Cf21Sz3&type=batch", request, String.class);
|
||||
System.out.println("推送URL结果：code:" + stringResponseEntity.getStatusCode().value() + ",body:" + stringResponseEntity.getBody());
|
||||
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
log.info(e.getMessage(), e);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
public List<Integer> queryPreAndNextIndexNum(Long bookId, Integer indexNum) {
|
||||
List<Integer> result = new ArrayList<>();
|
||||
|
||||
@ -91,7 +91,7 @@ public class CrawlBooksSchedule {
|
||||
}
|
||||
|
||||
private void updateBiquTaBooks(int finalI) {
|
||||
String baseUrl = "https://m.biquta.com";
|
||||
String baseUrl = "https://m.biquta.la";
|
||||
String catBookListUrlBase = baseUrl + "/class/";
|
||||
|
||||
int page = 1;//起始页码
|
||||
@ -167,28 +167,7 @@ public class CrawlBooksSchedule {
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
|
||||
ResponseEntity<Resource> resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = resEntity.getBody().getInputStream();
|
||||
picSrc = "/localPic/" + updateTimeStr.substring(0,4)+"/"+updateTimeStr.substring(5,7)+"/"+updateTimeStr.substring(8,10)
|
||||
+ UUIDUtils.getUUID32()
|
||||
+ picSrc.substring(picSrc.lastIndexOf("."));
|
||||
File picFile = new File(picSavePath+picSrc);
|
||||
File parentFile = picFile.getParentFile();
|
||||
if(!parentFile.exists()){
|
||||
parentFile.mkdirs();
|
||||
}
|
||||
OutputStream out = new FileOutputStream(picFile);
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1;) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
out.close();
|
||||
input.close();
|
||||
|
||||
}
|
||||
|
||||
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
@ -343,15 +322,15 @@ public class CrawlBooksSchedule {
|
||||
|
||||
//①爬分类列表的书籍url和总页数
|
||||
// https:
|
||||
////m.biquta.com/class/1/1.html
|
||||
////m.biquta.la/class/1/1.html
|
||||
// https:
|
||||
////m.biquta.com/class/2/1.html
|
||||
////m.biquta.la/class/2/1.html
|
||||
// https:
|
||||
////m.biquta.com/class/2/2.html
|
||||
////m.biquta.la/class/2/2.html
|
||||
//
|
||||
//
|
||||
// https:
|
||||
////m.biquta.com/class/2/2.html
|
||||
////m.biquta.la/class/2/2.html
|
||||
//<input type = "text" class="page_txt" value = "2/1244" size = "5" name = "txtPage" id = "txtPage" >
|
||||
//
|
||||
//
|
||||
@ -528,29 +507,6 @@ public class CrawlBooksSchedule {
|
||||
if (picMather.find()) {
|
||||
String picSrc = picMather.group(1);
|
||||
|
||||
if(picSaveType == 2 && StringUtils.isNotBlank(picSrc)){
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.add("Referer","https://www.biqudao.com");
|
||||
HttpEntity<String> requestEntity = new HttpEntity<>(null, headers);
|
||||
ResponseEntity<Resource> resEntity = isoRestTemplate.exchange(picSrc, HttpMethod.GET, requestEntity, Resource.class);
|
||||
InputStream input = resEntity.getBody().getInputStream();
|
||||
picSrc = "/localPic/" + updateTimeStr.substring(0,2)+"/"+updateTimeStr.substring(3,5)+"/"+updateTimeStr.substring(6,8)
|
||||
+ UUIDUtils.getUUID32()
|
||||
+ picSrc.substring(picSrc.lastIndexOf("."));
|
||||
File picFile = new File(picSavePath+picSrc);
|
||||
File parentFile = picFile.getParentFile();
|
||||
if(!parentFile.exists()){
|
||||
parentFile.mkdirs();
|
||||
}
|
||||
OutputStream out = new FileOutputStream(picFile);
|
||||
byte[] b = new byte[4096];
|
||||
for (int n; (n = input.read(b)) != -1;) {
|
||||
out.write(b, 0, n);
|
||||
}
|
||||
out.close();
|
||||
input.close();
|
||||
|
||||
}
|
||||
|
||||
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
|
||||
Matcher descMatch = descPatten.matcher(body);
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user