201206030/novel
1
0
Fork 0
mirror of https://github.com/201206030/novel.git synced 2025-07-16 17:26:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

集成后台管理系统，上线智能爬虫程序

Browse Source
This commit is contained in:
xxy
2019-11-15 07:15:20 +08:00
parent 7b861d2fb0
commit 53cff722d1
1125 changed files with 275618 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
novel-admin/src/main/java/com/java2nb/AdminApplication.java Normal file
View File

@ -0,0 +1,28 @@
package com.java2nb;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.ServletComponentScan;
import org.springframework.cache.annotation.EnableCaching;
import org.springframework.scheduling.annotation.EnableAsync;
import org.springframework.transaction.annotation.EnableTransactionManagement;
//关闭SpringSecurity的功能
@EnableAutoConfiguration(exclude = {
org.springframework.boot.autoconfigure.security.servlet.SecurityAutoConfiguration.class
})
@EnableTransactionManagement
@ServletComponentScan
@MapperScan("com.java2nb.*.dao")
@SpringBootApplication
@EnableCaching
public class AdminApplication {
public static void main(String[] args){
SpringApplication.run(AdminApplication.class);
}
}

128
novel-admin/src/main/java/com/java2nb/books/controller/BookContentController.java Normal file
View File

@ -0,0 +1,128 @@
package com.java2nb.books.controller;
import java.util.List;
import java.util.Map;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
import com.java2nb.books.domain.BookContentDO;
import com.java2nb.books.service.BookContentService;
import com.java2nb.common.utils.PageBean;
import com.java2nb.common.utils.Query;
import com.java2nb.common.utils.R;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:11
*/
@Controller
@RequestMapping("/books/bookContent")
public class BookContentController {
@Autowired
private BookContentService bookContentService;
@GetMapping()
@RequiresPermissions("books:bookContent:bookContent")
String BookContent() {
return "books/bookContent/bookContent";
}
@ApiOperation(value = "获取列表", notes = "获取列表")
@ResponseBody
@GetMapping("/list")
@RequiresPermissions("books:bookContent:bookContent")
public R list(@RequestParam Map<String, Object> params) {
//查询列表数据
Query query = new Query(params);
List<BookContentDO> bookContentList = bookContentService.list(query);
int total = bookContentService.count(query);
PageBean pageBean = new PageBean(bookContentList, total);
return R.ok().put("data", pageBean);
}
@ApiOperation(value = "新增页面", notes = "新增页面")
@GetMapping("/add")
String add() {
return "books/bookContent/add";
}
@ApiOperation(value = "修改页面", notes = "修改页面")
@GetMapping("/edit/{id}")
String edit(@PathVariable("id") Long id, Model model) {
BookContentDO bookContent = bookContentService.get(id);
model.addAttribute("bookContent", bookContent);
return "books/bookContent/edit";
}
@ApiOperation(value = "查看页面", notes = "查看页面")
@GetMapping("/detail/{id}")
String detail(@PathVariable("id") Long id, Model model) {
BookContentDO bookContent = bookContentService.get(id);
model.addAttribute("bookContent", bookContent);
return "books/bookContent/detail";
}
/**
* 保存
*/
@ApiOperation(value = "新增", notes = "新增")
@ResponseBody
@PostMapping("/save")
public R save( BookContentDO bookContent) {
if (bookContentService.save(bookContent) > 0) {
return R.ok();
}
return R.error();
}
/**
* 修改
*/
@ApiOperation(value = "修改", notes = "修改")
@ResponseBody
@RequestMapping("/update")
public R update( BookContentDO bookContent) {
bookContentService.update(bookContent);
return R.ok();
}
/**
* 删除
*/
@ApiOperation(value = "删除", notes = "删除")
@PostMapping("/remove")
@ResponseBody
public R remove( Long id) {
if (bookContentService.remove(id) > 0) {
return R.ok();
}
return R.error();
}
/**
* 删除
*/
@ApiOperation(value = "批量删除", notes = "批量删除")
@PostMapping("/batchRemove")
@ResponseBody
public R remove(@RequestParam("ids[]") Long[] ids) {
bookContentService.batchRemove(ids);
return R.ok();
}
}

128
novel-admin/src/main/java/com/java2nb/books/controller/BookController.java Normal file
View File

@ -0,0 +1,128 @@
package com.java2nb.books.controller;
import java.util.List;
import java.util.Map;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
import com.java2nb.books.domain.BookDO;
import com.java2nb.books.service.BookService;
import com.java2nb.common.utils.PageBean;
import com.java2nb.common.utils.Query;
import com.java2nb.common.utils.R;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:27:04
*/
@Controller
@RequestMapping("/books/book")
public class BookController {
@Autowired
private BookService bookService;
@GetMapping()
@RequiresPermissions("books:book:book")
String Book() {
return "books/book/book";
}
@ApiOperation(value = "获取列表", notes = "获取列表")
@ResponseBody
@GetMapping("/list")
@RequiresPermissions("books:book:book")
public R list(@RequestParam Map<String, Object> params) {
//查询列表数据
Query query = new Query(params);
List<BookDO> bookList = bookService.list(query);
int total = bookService.count(query);
PageBean pageBean = new PageBean(bookList, total);
return R.ok().put("data", pageBean);
}
@ApiOperation(value = "新增页面", notes = "新增页面")
@GetMapping("/add")
String add() {
return "books/book/add";
}
@ApiOperation(value = "修改页面", notes = "修改页面")
@GetMapping("/edit/{id}")
String edit(@PathVariable("id") Long id, Model model) {
BookDO book = bookService.get(id);
model.addAttribute("book", book);
return "books/book/edit";
}
@ApiOperation(value = "查看页面", notes = "查看页面")
@GetMapping("/detail/{id}")
String detail(@PathVariable("id") Long id, Model model) {
BookDO book = bookService.get(id);
model.addAttribute("book", book);
return "books/book/detail";
}
/**
* 保存
*/
@ApiOperation(value = "新增", notes = "新增")
@ResponseBody
@PostMapping("/save")
public R save( BookDO book) {
if (bookService.save(book) > 0) {
return R.ok();
}
return R.error();
}
/**
* 修改
*/
@ApiOperation(value = "修改", notes = "修改")
@ResponseBody
@RequestMapping("/update")
public R update( BookDO book) {
bookService.update(book);
return R.ok();
}
/**
* 删除
*/
@ApiOperation(value = "删除", notes = "删除")
@PostMapping("/remove")
@ResponseBody
public R remove( Long id) {
if (bookService.remove(id) > 0) {
return R.ok();
}
return R.error();
}
/**
* 删除
*/
@ApiOperation(value = "批量删除", notes = "批量删除")
@PostMapping("/batchRemove")
@ResponseBody
public R remove(@RequestParam("ids[]") Long[] ids) {
bookService.batchRemove(ids);
return R.ok();
}
}

145
novel-admin/src/main/java/com/java2nb/books/controller/BookCrawlController.java Normal file
View File

@ -0,0 +1,145 @@
package com.java2nb.books.controller;
import java.util.List;
import java.util.Map;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
import com.java2nb.books.domain.BookCrawlDO;
import com.java2nb.books.service.BookCrawlService;
import com.java2nb.common.utils.PageBean;
import com.java2nb.common.utils.Query;
import com.java2nb.common.utils.R;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-15 03:42:54
*/
@Controller
@RequestMapping("/books/bookCrawl")
public class BookCrawlController {
@Autowired
private BookCrawlService bookCrawlService;
@GetMapping()
@RequiresPermissions("books:bookCrawl:bookCrawl")
String BookCrawl() {
return "books/bookCrawl/bookCrawl";
}
@ApiOperation(value = "获取列表", notes = "获取列表")
@ResponseBody
@GetMapping("/list")
@RequiresPermissions("books:bookCrawl:bookCrawl")
public R list(@RequestParam Map<String, Object> params) {
//查询列表数据
Query query = new Query(params);
List<BookCrawlDO> bookCrawlList = bookCrawlService.list(query);
int total = bookCrawlService.count(query);
PageBean pageBean = new PageBean(bookCrawlList, total);
return R.ok().put("data", pageBean);
}
@ApiOperation(value = "新增页面", notes = "新增页面")
@GetMapping("/add")
@RequiresPermissions("books:bookCrawl:add")
String add() {
return "books/bookCrawl/add";
}
@ApiOperation(value = "修改页面", notes = "修改页面")
@GetMapping("/edit/{id}")
@RequiresPermissions("books:bookCrawl:edit")
String edit(@PathVariable("id") Long id, Model model) {
BookCrawlDO bookCrawl = bookCrawlService.get(id);
model.addAttribute("bookCrawl", bookCrawl);
return "books/bookCrawl/edit";
}
@ApiOperation(value = "查看页面", notes = "查看页面")
@GetMapping("/detail/{id}")
@RequiresPermissions("books:bookCrawl:detail")
String detail(@PathVariable("id") Long id, Model model) {
BookCrawlDO bookCrawl = bookCrawlService.get(id);
model.addAttribute("bookCrawl", bookCrawl);
return "books/bookCrawl/detail";
}
/**
* 保存
*/
@ApiOperation(value = "新增", notes = "新增")
@ResponseBody
@PostMapping("/save")
@RequiresPermissions("books:bookCrawl:add")
public R save( BookCrawlDO bookCrawl) {
if (bookCrawlService.save(bookCrawl) > 0) {
return R.ok();
}
return R.error();
}
/**
* 修改
*/
@ApiOperation(value = "修改", notes = "修改")
@ResponseBody
@RequestMapping("/update")
@RequiresPermissions("books:bookCrawl:edit")
public R update( BookCrawlDO bookCrawl) {
bookCrawlService.update(bookCrawl);
return R.ok();
}
/**
* 删除
*/
@ApiOperation(value = "删除", notes = "删除")
@PostMapping("/remove")
@ResponseBody
@RequiresPermissions("books:bookCrawl:remove")
public R remove( Long id) {
if (bookCrawlService.remove(id) > 0) {
return R.ok();
}
return R.error();
}
/**
* 删除
*/
@ApiOperation(value = "批量删除", notes = "批量删除")
@PostMapping("/batchRemove")
@ResponseBody
@RequiresPermissions("books:bookCrawl:batchRemove")
public R remove(@RequestParam("ids[]") Long[] ids) {
bookCrawlService.batchRemove(ids);
return R.ok();
}
/**
* 修改爬虫状态
*/
@ApiOperation(value = "修改爬虫状态", notes = "修改爬虫状态")
@ResponseBody
@RequestMapping("/updateStatus")
public R updateStatus( BookCrawlDO bookCrawl) {
bookCrawlService.updateStatus(bookCrawl);
return R.ok();
}
}

128
novel-admin/src/main/java/com/java2nb/books/controller/BookIndexController.java Normal file
View File

@ -0,0 +1,128 @@
package com.java2nb.books.controller;
import java.util.List;
import java.util.Map;
import org.apache.shiro.authz.annotation.RequiresPermissions;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.ResponseBody;
import io.swagger.annotations.ApiOperation;
import com.java2nb.books.domain.BookIndexDO;
import com.java2nb.books.service.BookIndexService;
import com.java2nb.common.utils.PageBean;
import com.java2nb.common.utils.Query;
import com.java2nb.common.utils.R;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:15
*/
@Controller
@RequestMapping("/books/bookIndex")
public class BookIndexController {
@Autowired
private BookIndexService bookIndexService;
@GetMapping()
@RequiresPermissions("books:bookIndex:bookIndex")
String BookIndex() {
return "books/bookIndex/bookIndex";
}
@ApiOperation(value = "获取列表", notes = "获取列表")
@ResponseBody
@GetMapping("/list")
@RequiresPermissions("books:bookIndex:bookIndex")
public R list(@RequestParam Map<String, Object> params) {
//查询列表数据
Query query = new Query(params);
List<BookIndexDO> bookIndexList = bookIndexService.list(query);
int total = bookIndexService.count(query);
PageBean pageBean = new PageBean(bookIndexList, total);
return R.ok().put("data", pageBean);
}
@ApiOperation(value = "新增页面", notes = "新增页面")
@GetMapping("/add")
String add() {
return "books/bookIndex/add";
}
@ApiOperation(value = "修改页面", notes = "修改页面")
@GetMapping("/edit/{id}")
String edit(@PathVariable("id") Long id, Model model) {
BookIndexDO bookIndex = bookIndexService.get(id);
model.addAttribute("bookIndex", bookIndex);
return "books/bookIndex/edit";
}
@ApiOperation(value = "查看页面", notes = "查看页面")
@GetMapping("/detail/{id}")
String detail(@PathVariable("id") Long id, Model model) {
BookIndexDO bookIndex = bookIndexService.get(id);
model.addAttribute("bookIndex", bookIndex);
return "books/bookIndex/detail";
}
/**
* 保存
*/
@ApiOperation(value = "新增", notes = "新增")
@ResponseBody
@PostMapping("/save")
public R save( BookIndexDO bookIndex) {
if (bookIndexService.save(bookIndex) > 0) {
return R.ok();
}
return R.error();
}
/**
* 修改
*/
@ApiOperation(value = "修改", notes = "修改")
@ResponseBody
@RequestMapping("/update")
public R update( BookIndexDO bookIndex) {
bookIndexService.update(bookIndex);
return R.ok();
}
/**
* 删除
*/
@ApiOperation(value = "删除", notes = "删除")
@PostMapping("/remove")
@ResponseBody
public R remove( Long id) {
if (bookIndexService.remove(id) > 0) {
return R.ok();
}
return R.error();
}
/**
* 删除
*/
@ApiOperation(value = "批量删除", notes = "批量删除")
@PostMapping("/batchRemove")
@ResponseBody
public R remove(@RequestParam("ids[]") Long[] ids) {
bookIndexService.batchRemove(ids);
return R.ok();
}
}

34
novel-admin/src/main/java/com/java2nb/books/dao/BookContentDao.java Normal file
View File

@ -0,0 +1,34 @@
package com.java2nb.books.dao;
import com.java2nb.books.domain.BookContentDO;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Mapper;
/**
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:11
*/
@Mapper
public interface BookContentDao {
BookContentDO get(Long id);
List<BookContentDO> list(Map<String,Object> map);
int count(Map<String,Object> map);
int save(BookContentDO bookContent);
int update(BookContentDO bookContent);
int remove(Long id);
int batchRemove(Long[] ids);
void insertBatch(List<BookContentDO> newContentList);
}

34
novel-admin/src/main/java/com/java2nb/books/dao/BookCrawlDao.java Normal file
View File

@ -0,0 +1,34 @@
package com.java2nb.books.dao;
import com.java2nb.books.domain.BookCrawlDO;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Mapper;
/**
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-15 03:42:54
*/
@Mapper
public interface BookCrawlDao {
BookCrawlDO get(Long id);
List<BookCrawlDO> list(Map<String,Object> map);
int count(Map<String,Object> map);
int save(BookCrawlDO bookCrawl);
int update(BookCrawlDO bookCrawl);
int remove(Long id);
int batchRemove(Long[] ids);
void initStatus();
}

32
novel-admin/src/main/java/com/java2nb/books/dao/BookDao.java Normal file
View File

@ -0,0 +1,32 @@
package com.java2nb.books.dao;
import com.java2nb.books.domain.BookDO;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Mapper;
/**
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:27:04
*/
@Mapper
public interface BookDao {
BookDO get(Long id);
List<BookDO> list(Map<String,Object> map);
int count(Map<String,Object> map);
int save(BookDO book);
int update(BookDO book);
int remove(Long id);
int batchRemove(Long[] ids);
}

34
novel-admin/src/main/java/com/java2nb/books/dao/BookIndexDao.java Normal file
View File

@ -0,0 +1,34 @@
package com.java2nb.books.dao;
import com.java2nb.books.domain.BookIndexDO;
import java.util.List;
import java.util.Map;
import org.apache.ibatis.annotations.Mapper;
/**
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:15
*/
@Mapper
public interface BookIndexDao {
BookIndexDO get(Long id);
List<BookIndexDO> list(Map<String,Object> map);
int count(Map<String,Object> map);
int save(BookIndexDO bookIndex);
int update(BookIndexDO bookIndex);
int remove(Long id);
int batchRemove(Long[] ids);
void insertBatch(List<BookIndexDO> newBookIndexList);
}

104
novel-admin/src/main/java/com/java2nb/books/domain/BookContentDO.java Normal file
View File

@ -0,0 +1,104 @@
package com.java2nb.books.domain;
import java.io.Serializable;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.java2nb.common.jsonserializer.LongToStringSerializer;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:11
*/
public class BookContentDO implements Serializable {
private static final long serialVersionUID = 1L;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long id;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long bookId;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long indexId;
//
private Integer indexNum;
//
private String content;
/**
* 设置：
*/
public void setId(Long id) {
this.id = id;
}
/**
* 获取：
*/
public Long getId() {
return id;
}
/**
* 设置：
*/
public void setBookId(Long bookId) {
this.bookId = bookId;
}
/**
* 获取：
*/
public Long getBookId() {
return bookId;
}
/**
* 设置：
*/
public void setIndexId(Long indexId) {
this.indexId = indexId;
}
/**
* 获取：
*/
public Long getIndexId() {
return indexId;
}
/**
* 设置：
*/
public void setIndexNum(Integer indexNum) {
this.indexNum = indexNum;
}
/**
* 获取：
*/
public Integer getIndexNum() {
return indexNum;
}
/**
* 设置：
*/
public void setContent(String content) {
this.content = content;
}
/**
* 获取：
*/
public String getContent() {
return content;
}
}

98
novel-admin/src/main/java/com/java2nb/books/domain/BookCrawlDO.java Normal file
View File

@ -0,0 +1,98 @@
package com.java2nb.books.domain;
import java.io.Serializable;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.java2nb.common.jsonserializer.LongToStringSerializer;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-15 03:42:54
*/
public class BookCrawlDO implements Serializable {
private static final long serialVersionUID = 1L;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long id;
//
private String crawlWebName;
//
private String crawlWebUrl;
//
private Integer crawlWebCode;
//
private Integer status;
/**
* 设置：
*/
public void setId(Long id) {
this.id = id;
}
/**
* 获取：
*/
public Long getId() {
return id;
}
/**
* 设置：
*/
public void setCrawlWebName(String crawlWebName) {
this.crawlWebName = crawlWebName;
}
/**
* 获取：
*/
public String getCrawlWebName() {
return crawlWebName;
}
/**
* 设置：
*/
public void setCrawlWebUrl(String crawlWebUrl) {
this.crawlWebUrl = crawlWebUrl;
}
/**
* 获取：
*/
public String getCrawlWebUrl() {
return crawlWebUrl;
}
/**
* 设置：
*/
public void setCrawlWebCode(Integer crawlWebCode) {
this.crawlWebCode = crawlWebCode;
}
/**
* 获取：
*/
public Integer getCrawlWebCode() {
return crawlWebCode;
}
/**
* 设置：
*/
public void setStatus(Integer status) {
this.status = status;
}
/**
* 获取：
*/
public Integer getStatus() {
return status;
}
}

202
novel-admin/src/main/java/com/java2nb/books/domain/BookDO.java Normal file
View File

@ -0,0 +1,202 @@
package com.java2nb.books.domain;
import java.io.Serializable;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.java2nb.common.jsonserializer.LongToStringSerializer;
import org.springframework.format.annotation.DateTimeFormat;
import java.util.Date;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:27:04
*/
public class BookDO implements Serializable {
private static final long serialVersionUID = 1L;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long id;
//
private Integer catid;
//
private String picUrl;
//
private String bookName;
//
private String author;
//
private String bookDesc;
//
private Float score;
//
private String bookStatus;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long visitCount;
//
@DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss")
private Date updateTime;
//
private Integer softCat;
//
private String softTag;
/**
* 设置：
*/
public void setId(Long id) {
this.id = id;
}
/**
* 获取：
*/
public Long getId() {
return id;
}
/**
* 设置：
*/
public void setCatid(Integer catid) {
this.catid = catid;
}
/**
* 获取：
*/
public Integer getCatid() {
return catid;
}
/**
* 设置：
*/
public void setPicUrl(String picUrl) {
this.picUrl = picUrl;
}
/**
* 获取：
*/
public String getPicUrl() {
return picUrl;
}
/**
* 设置：
*/
public void setBookName(String bookName) {
this.bookName = bookName;
}
/**
* 获取：
*/
public String getBookName() {
return bookName;
}
/**
* 设置：
*/
public void setAuthor(String author) {
this.author = author;
}
/**
* 获取：
*/
public String getAuthor() {
return author;
}
/**
* 设置：
*/
public void setBookDesc(String bookDesc) {
this.bookDesc = bookDesc;
}
/**
* 获取：
*/
public String getBookDesc() {
return bookDesc;
}
/**
* 设置：
*/
public void setScore(Float score) {
this.score = score;
}
/**
* 获取：
*/
public Float getScore() {
return score;
}
/**
* 设置：
*/
public void setBookStatus(String bookStatus) {
this.bookStatus = bookStatus;
}
/**
* 获取：
*/
public String getBookStatus() {
return bookStatus;
}
/**
* 设置：
*/
public void setVisitCount(Long visitCount) {
this.visitCount = visitCount;
}
/**
* 获取：
*/
public Long getVisitCount() {
return visitCount;
}
/**
* 设置：
*/
public void setUpdateTime(Date updateTime) {
this.updateTime = updateTime;
}
/**
* 获取：
*/
public Date getUpdateTime() {
return updateTime;
}
/**
* 设置：
*/
public void setSoftCat(Integer softCat) {
this.softCat = softCat;
}
/**
* 获取：
*/
public Integer getSoftCat() {
return softCat;
}
/**
* 设置：
*/
public void setSoftTag(String softTag) {
this.softTag = softTag;
}
/**
* 获取：
*/
public String getSoftTag() {
return softTag;
}
}

87
novel-admin/src/main/java/com/java2nb/books/domain/BookIndexDO.java Normal file
View File

@ -0,0 +1,87 @@
package com.java2nb.books.domain;
import java.io.Serializable;
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
import com.java2nb.common.jsonserializer.LongToStringSerializer;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:15
*/
public class BookIndexDO implements Serializable {
private static final long serialVersionUID = 1L;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long id;
//
//java中的long能表示的范围比js中number大,也就意味着部分数值在js中存不下(变成不准确的值)
//所以通过序列化成字符串来解决
@JsonSerialize(using = LongToStringSerializer.class)
private Long bookId;
//
private Integer indexNum;
//
private String indexName;
/**
* 设置：
*/
public void setId(Long id) {
this.id = id;
}
/**
* 获取：
*/
public Long getId() {
return id;
}
/**
* 设置：
*/
public void setBookId(Long bookId) {
this.bookId = bookId;
}
/**
* 获取：
*/
public Long getBookId() {
return bookId;
}
/**
* 设置：
*/
public void setIndexNum(Integer indexNum) {
this.indexNum = indexNum;
}
/**
* 获取：
*/
public Integer getIndexNum() {
return indexNum;
}
/**
* 设置：
*/
public void setIndexName(String indexName) {
this.indexName = indexName;
}
/**
* 获取：
*/
public String getIndexName() {
return indexName;
}
}

21
novel-admin/src/main/java/com/java2nb/books/listener/AppStartListener.java Normal file
View File

@ -0,0 +1,21 @@
package com.java2nb.books.listener;
import com.java2nb.books.dao.BookCrawlDao;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.ApplicationListener;
import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.stereotype.Component;
@Component
public class AppStartListener implements ApplicationListener<ContextRefreshedEvent> {
@Autowired
private BookCrawlDao bookCrawlDao;
@Override
public void onApplicationEvent(ContextRefreshedEvent contextRefreshedEvent) {
bookCrawlDao.initStatus();
System.out.println("项目启动成功");
}
}

30
novel-admin/src/main/java/com/java2nb/books/service/BookContentService.java Normal file
View File

@ -0,0 +1,30 @@
package com.java2nb.books.service;
import com.java2nb.books.domain.BookContentDO;
import java.util.List;
import java.util.Map;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:11
*/
public interface BookContentService {
BookContentDO get(Long id);
List<BookContentDO> list(Map<String, Object> map);
int count(Map<String, Object> map);
int save(BookContentDO bookContent);
int update(BookContentDO bookContent);
int remove(Long id);
int batchRemove(Long[] ids);
}

32
novel-admin/src/main/java/com/java2nb/books/service/BookCrawlService.java Normal file
View File

@ -0,0 +1,32 @@
package com.java2nb.books.service;
import com.java2nb.books.domain.BookCrawlDO;
import java.util.List;
import java.util.Map;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-15 03:42:54
*/
public interface BookCrawlService {
BookCrawlDO get(Long id);
List<BookCrawlDO> list(Map<String, Object> map);
int count(Map<String, Object> map);
int save(BookCrawlDO bookCrawl);
int update(BookCrawlDO bookCrawl);
int remove(Long id);
int batchRemove(Long[] ids);
void updateStatus(BookCrawlDO bookCrawl);
}

30
novel-admin/src/main/java/com/java2nb/books/service/BookIndexService.java Normal file
View File

@ -0,0 +1,30 @@
package com.java2nb.books.service;
import com.java2nb.books.domain.BookIndexDO;
import java.util.List;
import java.util.Map;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:28:15
*/
public interface BookIndexService {
BookIndexDO get(Long id);
List<BookIndexDO> list(Map<String, Object> map);
int count(Map<String, Object> map);
int save(BookIndexDO bookIndex);
int update(BookIndexDO bookIndex);
int remove(Long id);
int batchRemove(Long[] ids);
}

30
novel-admin/src/main/java/com/java2nb/books/service/BookService.java Normal file
View File

@ -0,0 +1,30 @@
package com.java2nb.books.service;
import com.java2nb.books.domain.BookDO;
import java.util.List;
import java.util.Map;
/**
*
*
* @author xiongxy
* @email 1179705413@qq.com
* @date 2019-11-13 09:27:04
*/
public interface BookService {
BookDO get(Long id);
List<BookDO> list(Map<String, Object> map);
int count(Map<String, Object> map);
int save(BookDO book);
int update(BookDO book);
int remove(Long id);
int batchRemove(Long[] ids);
}

55
novel-admin/src/main/java/com/java2nb/books/service/impl/BookContentServiceImpl.java Normal file
View File

@ -0,0 +1,55 @@
package com.java2nb.books.service.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import com.java2nb.books.dao.BookContentDao;
import com.java2nb.books.domain.BookContentDO;
import com.java2nb.books.service.BookContentService;
@Service
public class BookContentServiceImpl implements BookContentService {
@Autowired
private BookContentDao bookContentDao;
@Override
public BookContentDO get(Long id){
return bookContentDao.get(id);
}
@Override
public List<BookContentDO> list(Map<String, Object> map){
return bookContentDao.list(map);
}
@Override
public int count(Map<String, Object> map){
return bookContentDao.count(map);
}
@Override
public int save(BookContentDO bookContent){
return bookContentDao.save(bookContent);
}
@Override
public int update(BookContentDO bookContent){
return bookContentDao.update(bookContent);
}
@Override
public int remove(Long id){
return bookContentDao.remove(id);
}
@Override
public int batchRemove(Long[] ids){
return bookContentDao.batchRemove(ids);
}
}

701
novel-admin/src/main/java/com/java2nb/books/service/impl/BookCrawlServiceImpl.java Normal file
View File

@ -0,0 +1,701 @@
package com.java2nb.books.service.impl;
import com.java2nb.books.dao.BookContentDao;
import com.java2nb.books.dao.BookDao;
import com.java2nb.books.dao.BookIndexDao;
import com.java2nb.books.domain.BookContentDO;
import com.java2nb.books.domain.BookDO;
import com.java2nb.books.domain.BookIndexDO;
import com.java2nb.books.util.RestTemplateUtil;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Service;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.java2nb.books.dao.BookCrawlDao;
import com.java2nb.books.domain.BookCrawlDO;
import com.java2nb.books.service.BookCrawlService;
import org.springframework.web.client.RestTemplate;
@Service
public class BookCrawlServiceImpl implements BookCrawlService {
private boolean isInteruptBiquDaoCrawl;//是否中断笔趣岛爬虫程序
private boolean isInteruptBiquTaCrawl;//是否中断笔趣塔爬虫程序
private RestTemplate restTemplate = RestTemplateUtil.getInstance("utf-8");
@Autowired
private BookCrawlDao bookCrawlDao;
@Autowired
private BookDao bookDao;
@Autowired
private BookIndexDao bookIndexDao;
@Autowired
private BookContentDao bookContentDao;
@Override
public BookCrawlDO get(Long id){
return bookCrawlDao.get(id);
}
@Override
public List<BookCrawlDO> list(Map<String, Object> map){
return bookCrawlDao.list(map);
}
@Override
public int count(Map<String, Object> map){
return bookCrawlDao.count(map);
}
@Override
public int save(BookCrawlDO bookCrawl){
return bookCrawlDao.save(bookCrawl);
}
@Override
public int update(BookCrawlDO bookCrawl){
return bookCrawlDao.update(bookCrawl);
}
@Override
public int remove(Long id){
return bookCrawlDao.remove(id);
}
@Override
public int batchRemove(Long[] ids){
return bookCrawlDao.batchRemove(ids);
}
@Override
public void updateStatus(BookCrawlDO bookCrawl) {
bookCrawlDao.update(bookCrawl);
if(bookCrawl.getStatus() == 0){
switch (bookCrawl.getCrawlWebCode()) {
case 1: {
isInteruptBiquDaoCrawl = true;
break;
}
case 2: {
isInteruptBiquTaCrawl = true;
break;
}
}
}else{
crawlBook(bookCrawl.getCrawlWebCode());
}
}
private void crawlBook(int status){
for (int i = 1; i <= 7; i++) {
int finalI = i;
new Thread(
() -> {
try {
switch (status) {
case 1: {
crawBiqudaoBooks(finalI);
break;
}
case 2: {
crawBiquTaBooks(finalI);
break;
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
).start();
}
}
private void crawBiquTaBooks(int i) {
String baseUrl = "https://m.biquta.com";
String catBookListUrlBase = baseUrl + "/class/";
//拼接分类URL
int page = 1;//起始页码
int totalPage = page;
String catBookListUrl = catBookListUrlBase + i + "/" + page + ".html";
String forObject = getByHttpClient(catBookListUrl);
if (forObject != null) {
//匹配分页数<input type="text" class="page_txt" value="1/3019" size="5" name="txtPage" id="txtPage" />
Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\"");
Matcher matcher = pattern.matcher(forObject);
boolean isFind = matcher.find();
System.out.println("匹配分页数" + isFind);
if (isFind) {
int currentPage = Integer.parseInt(matcher.group(1));
totalPage = Integer.parseInt(matcher.group(2));
//解析第一页书籍的数据
Pattern bookPatten = Pattern.compile("href=\"/(\\d+_\\d+)/\"");
parseBiquTaBook(bookPatten, forObject, i, baseUrl);
while (currentPage < totalPage) {
if(isInteruptBiquTaCrawl){
break;
}
catBookListUrl = catBookListUrlBase + i + "/" + (currentPage + 1) + ".html";
forObject = getByHttpClient(catBookListUrl);
if (forObject != null) {
//匹配分页数
matcher = pattern.matcher(forObject);
isFind = matcher.find();
if (isFind) {
currentPage = Integer.parseInt(matcher.group(1));
totalPage = Integer.parseInt(matcher.group(2));
parseBiquTaBook(bookPatten, forObject, i, baseUrl);
}
} else {
currentPage++;
}
}
}
}
}
private void parseBiquTaBook(Pattern bookPatten, String forObject, int catNum, String baseUrl) {
Matcher matcher2 = bookPatten.matcher(forObject);
boolean isFind = matcher2.find();
Pattern scorePatten = Pattern.compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
Matcher scoreMatch = scorePatten.matcher(forObject);
boolean scoreFind = scoreMatch.find();
Pattern bookNamePatten = Pattern.compile("<p class=\"title\">([^/]+)</p>");
Matcher bookNameMatch = bookNamePatten.matcher(forObject);
boolean isBookNameMatch = bookNameMatch.find();
Pattern authorPatten = Pattern.compile(">作者：([^/]+)<");
Matcher authoreMatch = authorPatten.matcher(forObject);
boolean isFindAuthor = authoreMatch.find();
System.out.println("匹配书籍url" + isFind);
System.out.println("匹配分数" + scoreFind);
while (isFind && scoreFind && isBookNameMatch && isFindAuthor) {
if(isInteruptBiquTaCrawl){
break;
}
try {
Float score = Float.parseFloat(scoreMatch.group(1));
/*if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说
// Thread.sleep(1000 * 60 * 60 * 24);//因为爬的是龙虎榜，所以遇到第一个8分以下的，之后的都是8分以下的
continue;
}*/
String bookName = bookNameMatch.group(1);
String author = authoreMatch.group(1);
/*//查询该书籍是否存在
boolean isExsit = bookService.isExsitBook(bookName, author);
if (isExsit) {
continue;
}*/
//System.out.println(new Date()+bookName + "：");
String bokNum = matcher2.group(1);
String bookUrl = baseUrl + "/" + bokNum + "/";
String body = getByHttpClient(bookUrl);
if (body != null) {
Pattern statusPatten = Pattern.compile("状态：([^/]+)</li>");
Matcher statusMatch = statusPatten.matcher(body);
if (statusMatch.find()) {
String status = statusMatch.group(1);
Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
Matcher updateTimeMatch = updateTimePatten.matcher(body);
if (updateTimeMatch.find()) {
String updateTimeStr = updateTimeMatch.group(1);
SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
Date updateTime = format.parse(updateTimeStr);
Pattern picPatten = Pattern.compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
Matcher picMather = picPatten.matcher(body);
if (picMather.find()) {
String picSrc = picMather.group(1);
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
Matcher descMatch = descPatten.matcher(body);
if (descMatch.find()) {
String desc = descMatch.group(1);
BookDO book = new BookDO();
book.setAuthor(author);
book.setCatid(catNum);
book.setBookDesc(desc);
book.setBookName(bookName);
book.setScore(score > 10 ? 8.0f : score);
book.setPicUrl(picSrc);
book.setBookStatus(status);
book.setUpdateTime(updateTime);
List<BookIndexDO> indexList = new ArrayList<>();
List<BookContentDO> contentList = new ArrayList<>();
//读取目录
Pattern indexPatten = Pattern.compile("<a\\s+href=\"(/du/\\d+_\\d+/)\">查看完整目录</a>");
Matcher indexMatch = indexPatten.matcher(body);
if (indexMatch.find()) {
String indexUrl = baseUrl + indexMatch.group(1);
String body2 = getByHttpClient(indexUrl);
if (body2 != null) {
Pattern indexListPatten = Pattern.compile("<a\\s+style=\"\"\\s+href=\"(/\\d+_\\d+/\\d+\\.html)\">([^/]+)</a>");
Matcher indexListMatch = indexListPatten.matcher(body2);
boolean isFindIndex = indexListMatch.find();
int indexNum = 0;
//查询该书籍已存在目录号
List<Integer> hasIndexNum = queryIndexCountByBookNameAndBAuthor(bookName, author);
while (isFindIndex) {
if(isInteruptBiquTaCrawl){
break;
}
if (!hasIndexNum.contains(indexNum)) {
String contentUrl = baseUrl + indexListMatch.group(1);
String indexName = indexListMatch.group(2);
//查询章节内容
String body3 = getByHttpClient(contentUrl);
if (body3 != null) {
Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读");
String start = "『章节错误,点此举报』";
String end = "『加入书签，方便阅读』";
String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
//TODO插入章节目录和章节内容
BookIndexDO bookIndex = new BookIndexDO();
bookIndex.setIndexName(indexName);
bookIndex.setIndexNum(indexNum);
indexList.add(bookIndex);
BookContentDO bookContent = new BookContentDO();
bookContent.setContent(content);
bookContent.setIndexNum(indexNum);
contentList.add(bookContent);
//System.out.println(indexName);
} else {
break;
}
}
indexNum++;
isFindIndex = indexListMatch.find();
}
if (indexList.size() == contentList.size() && indexList.size() > 0) {
saveBookAndIndexAndContent(book, indexList, contentList);
}
}
}
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
matcher2.find();
isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配
scoreFind = scoreMatch.find();
isBookNameMatch = bookNameMatch.find();
isFindAuthor = authoreMatch.find();
}
}
}
private void crawBiqudaoBooks(final int i) {
String baseUrl = "https://m.biqudao.com";
String catBookListUrlBase = baseUrl + "/bqgelhb/";
//拼接分类URL
int page = 1;//起始页码
int totalPage = page;
String catBookListUrl = catBookListUrlBase + i + "/" + page + ".html";
String forObject = getByHttpClient(catBookListUrl);
if (forObject != null) {
//匹配分页数<input type="text" class="page_txt" value="1/3019" size="5" name="txtPage" id="txtPage" />
Pattern pattern = Pattern.compile("value=\"(\\d+)/(\\d+)\"");
Matcher matcher = pattern.matcher(forObject);
boolean isFind = matcher.find();
System.out.println("匹配分页数" + isFind);
if (isFind) {
int currentPage = Integer.parseInt(matcher.group(1));
totalPage = Integer.parseInt(matcher.group(2));
//解析第一页书籍的数据
Pattern bookPatten = Pattern.compile("href=\"/(bqge\\d+)/\"");
parseBiqudaoBook(bookPatten, forObject, i, baseUrl);
while (currentPage < totalPage) {
if(isInteruptBiquDaoCrawl){
break;
}
catBookListUrl = catBookListUrlBase + i + "/" + (currentPage + 1) + ".html";
forObject = getByHttpClient(catBookListUrl);
if (forObject != null) {
//匹配分页数
matcher = pattern.matcher(forObject);
isFind = matcher.find();
if (isFind) {
currentPage = Integer.parseInt(matcher.group(1));
totalPage = Integer.parseInt(matcher.group(2));
parseBiqudaoBook(bookPatten, forObject, i, baseUrl);
}
} else {
currentPage++;
}
}
}
}
}
private void parseBiqudaoBook(Pattern bookPatten, String forObject, int catNum, String baseUrl) {
Matcher matcher2 = bookPatten.matcher(forObject);
boolean isFind = matcher2.find();
Pattern scorePatten = Pattern.compile("<div\\s+class=\"score\">(\\d+\\.\\d+)分</div>");
Matcher scoreMatch = scorePatten.matcher(forObject);
boolean scoreFind = scoreMatch.find();
Pattern bookNamePatten = Pattern.compile("<p class=\"title\">([^/]+)</p>");
Matcher bookNameMatch = bookNamePatten.matcher(forObject);
boolean isBookNameMatch = bookNameMatch.find();
Pattern authorPatten = Pattern.compile(">作者：([^<]+)<");
Matcher authoreMatch = authorPatten.matcher(forObject);
boolean isFindAuthor = authoreMatch.find();
System.out.println("匹配书籍url" + isFind);
System.out.println("匹配分数" + scoreFind);
while (isFind && scoreFind && isBookNameMatch && isFindAuthor) {
try {
if(isInteruptBiquDaoCrawl){
break;
}
Float score = Float.parseFloat(scoreMatch.group(1));
/*if (score < lowestScore) {//数据库空间有限，暂时爬取8.0分以上的小说
Thread.sleep(1000 * 60 * 60 * 24);//因为爬的是龙虎榜，所以遇到第一个8分以下的，之后的都是8分以下的
continue;
}*/
String bookName = bookNameMatch.group(1);
String author = authoreMatch.group(1);
/*//查询该书籍是否存在
boolean isExsit = bookService.isExsitBook(bookName, author);
if (isExsit) {
continue;
}*/
//System.out.println(new Date()+bookName + "：");
String bokNum = matcher2.group(1);
String bookUrl = baseUrl + "/" + bokNum + "/";
String body = getByHttpClient(bookUrl);
if (body != null) {
Pattern statusPatten = Pattern.compile("状态：([^/]+)</li>");
Matcher statusMatch = statusPatten.matcher(body);
if (statusMatch.find()) {
String status = statusMatch.group(1);
Pattern updateTimePatten = Pattern.compile("更新：(\\d+-\\d+-\\d+\\s\\d+:\\d+:\\d+)</a>");
Matcher updateTimeMatch = updateTimePatten.matcher(body);
if (updateTimeMatch.find()) {
String updateTimeStr = updateTimeMatch.group(1);
SimpleDateFormat format = new SimpleDateFormat("yy-MM-dd HH:mm:ss");
Date updateTime = format.parse(updateTimeStr);
Pattern picPatten = Pattern.compile("<img src=\"([^>]+)\"\\s+onerror=\"this.src=");
Matcher picMather = picPatten.matcher(body);
if (picMather.find()) {
String picSrc = picMather.group(1);
Pattern descPatten = Pattern.compile("class=\"review\">([^<]+)</p>");
Matcher descMatch = descPatten.matcher(body);
if (descMatch.find()) {
String desc = descMatch.group(1);
BookDO book = new BookDO();
book.setAuthor(author);
book.setCatid(catNum);
book.setBookDesc(desc);
book.setBookName(bookName);
book.setScore(score > 10 ? 8.0f : score);
book.setPicUrl(picSrc);
book.setBookStatus(status);
book.setUpdateTime(updateTime);
List<BookIndexDO> indexList = new ArrayList<>();
List<BookContentDO> contentList = new ArrayList<>();
//读取目录
Pattern indexPatten = Pattern.compile("<a\\s+href=\"(/bqge\\d+/all\\.html)\">查看完整目录</a>");
Matcher indexMatch = indexPatten.matcher(body);
if (indexMatch.find()) {
String indexUrl = baseUrl + indexMatch.group(1);
String body2 = getByHttpClient(indexUrl);
if (body2 != null) {
Pattern indexListPatten = Pattern.compile("<a[^/]+style[^/]+href=\"(/bqge\\d+/\\d+\\.html)\">([^/]+)</a>");
Matcher indexListMatch = indexListPatten.matcher(body2);
boolean isFindIndex = indexListMatch.find();
int indexNum = 0;
//查询该书籍已存在目录号
List<Integer> hasIndexNum = queryIndexCountByBookNameAndBAuthor(bookName, author);
while (isFindIndex) {
if(isInteruptBiquDaoCrawl){
break;
}
if (!hasIndexNum.contains(indexNum)) {
String contentUrl = baseUrl + indexListMatch.group(1);
String indexName = indexListMatch.group(2);
//查询章节内容
String body3 = getByHttpClient(contentUrl);
if (body3 != null) {
Pattern contentPattten = Pattern.compile("章节错误,点此举报(.*)加入书签，方便阅读");
String start = "『章节错误,点此举报』";
String end = "『加入书签，方便阅读』";
String content = body3.substring(body3.indexOf(start) + start.length(), body3.indexOf(end));
//TODO插入章节目录和章节内容
BookIndexDO bookIndex = new BookIndexDO();
bookIndex.setIndexName(indexName);
bookIndex.setIndexNum(indexNum);
indexList.add(bookIndex);
BookContentDO bookContent = new BookContentDO();
bookContent.setContent(content);
bookContent.setIndexNum(indexNum);
contentList.add(bookContent);
//System.out.println(indexName);
} else {
break;
}
}
indexNum++;
isFindIndex = indexListMatch.find();
}
if (indexList.size() == contentList.size() && indexList.size() > 0) {
saveBookAndIndexAndContent(book, indexList, contentList);
}
}
}
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
matcher2.find();
isFind = matcher2.find();//需要找两次，应为有两个一样的路径匹配
scoreFind = scoreMatch.find();
isBookNameMatch = bookNameMatch.find();
isFindAuthor = authoreMatch.find();
}
}
}
public void saveBookAndIndexAndContent(BookDO book, List<BookIndexDO> bookIndex, List<BookContentDO> bookContent) {
boolean isUpdate = false;
Long bookId = -1l;
book.setBookName(book.getBookName().trim());
book.setAuthor(book.getAuthor().trim());
Map<String,Object> bookExample = new HashMap<>();
bookExample.put("bookName",book.getBookName());
bookExample.put("author",book.getAuthor());
List<BookDO> books = bookDao.list(bookExample);
if (books.size() > 0) {
//更新
bookId = books.get(0).getId();
book.setId(bookId);
bookDao.update(book);
isUpdate = true;
} else {
if (book.getVisitCount() == null) {
Long visitCount = generateVisiteCount(book.getScore());
book.setVisitCount(visitCount);
}
//插入
int rows = bookDao.save(book);
if (rows > 0) {
bookId = book.getId();
}
}
if (bookId >= 0) {
//查询目录已存在数量
/* BookIndexExample bookIndexExample = new BookIndexExample();
bookIndexExample.createCriteria().andBookIdEqualTo(bookId);
int indexCount = bookIndexMapper.countByExample(bookIndexExample);*/
List<BookIndexDO> newBookIndexList = new ArrayList<>();
List<BookContentDO> newContentList = new ArrayList<>();
for (int i = 0; i < bookIndex.size(); i++) {
BookContentDO bookContentItem = bookContent.get(i);
if (!bookContentItem.getContent().contains("正在手打中，请稍等片刻，内容更新后，需要重新刷新页面，才能获取最新更新")) {
BookIndexDO bookIndexItem = bookIndex.get(i);
bookIndexItem.setBookId(bookId);
bookContentItem.setBookId(bookId);
//bookContentItem.setIndexId(bookIndexItem.getId());暂时使用bookId和IndexNum查询content
bookContentItem.setIndexNum(bookIndexItem.getIndexNum());
newBookIndexList.add(bookIndexItem);
newContentList.add(bookContentItem);
}
}
if (newBookIndexList.size() > 0) {
bookIndexDao.insertBatch(newBookIndexList);
bookContentDao.insertBatch(newContentList);
}
}
}
private Long generateVisiteCount(Float score) {
int baseNum = (int) (Math.pow(score * 10, (int) (score - 5)) / 2);
return Long.parseLong(baseNum + new Random().nextInt(1000) + "");
}
private String getByHttpClient(String catBookListUrl) {
try {
/*HttpClient httpClient = new DefaultHttpClient();
// 设置请求和传输超时时间
RequestConfig requestConfig = RequestConfig.custom().setSocketTimeout(30000).setConnectTimeout(30000)
.setRedirectsEnabled(false) // 不自动重定向
.build();
HttpGet getReq = new HttpGet(catBookListUrl);
getReq.setConfig(requestConfig);
getReq.setHeader("user-agent", "Mozilla/5.0 (iPad; CPU OS 11_0 like Mac OS X) AppleWebKit/604.1.34 (KHTML, like Gecko) Version/11.0 Mobile/15A5341f Safari/604.1");
HttpResponse execute = httpClient.execute(getReq);
if (execute.getStatusLine().getStatusCode() == HttpStatus.OK.value()) {
HttpEntity entity = execute.getEntity();
return EntityUtils.toString(entity, "utf-8");
} else {
return null;
}*/
//经测试restTemplate比httpClient效率高出很多倍，所有选择restTemplate
ResponseEntity<String> forEntity = restTemplate.getForEntity(catBookListUrl, String.class);
if (forEntity.getStatusCode() == HttpStatus.OK) {
return forEntity.getBody();
} else {
return null;
}
} catch (Exception e) {
e.printStackTrace();
return null;
}
}
/**
* 查询该书籍目录数量
*/
private List<Integer> queryIndexCountByBookNameAndBAuthor(String bookName, String author) {
List<Integer> result = new ArrayList<>();
Map<String,Object> bookExample = new HashMap<>();
bookExample.put("bookName",bookName);
bookExample.put("author",author);
List<BookDO> books = bookDao.list(bookExample);
if (books.size() > 0) {
Long bookId = books.get(0).getId();
Map<String,Object> bookIndexExample = new HashMap<>();
bookExample.put("bookId",bookId);
List<BookIndexDO> bookIndices = bookIndexDao.list(bookIndexExample);
if (bookIndices != null && bookIndices.size() > 0) {
for (BookIndexDO bookIndex : bookIndices) {
result.add(bookIndex.getIndexNum());
}
}
}
return result;
}
}

55
novel-admin/src/main/java/com/java2nb/books/service/impl/BookIndexServiceImpl.java Normal file
View File

@ -0,0 +1,55 @@
package com.java2nb.books.service.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import com.java2nb.books.dao.BookIndexDao;
import com.java2nb.books.domain.BookIndexDO;
import com.java2nb.books.service.BookIndexService;
@Service
public class BookIndexServiceImpl implements BookIndexService {
@Autowired
private BookIndexDao bookIndexDao;
@Override
public BookIndexDO get(Long id){
return bookIndexDao.get(id);
}
@Override
public List<BookIndexDO> list(Map<String, Object> map){
return bookIndexDao.list(map);
}
@Override
public int count(Map<String, Object> map){
return bookIndexDao.count(map);
}
@Override
public int save(BookIndexDO bookIndex){
return bookIndexDao.save(bookIndex);
}
@Override
public int update(BookIndexDO bookIndex){
return bookIndexDao.update(bookIndex);
}
@Override
public int remove(Long id){
return bookIndexDao.remove(id);
}
@Override
public int batchRemove(Long[] ids){
return bookIndexDao.batchRemove(ids);
}
}

55
novel-admin/src/main/java/com/java2nb/books/service/impl/BookServiceImpl.java Normal file
View File

@ -0,0 +1,55 @@
package com.java2nb.books.service.impl;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.List;
import java.util.Map;
import com.java2nb.books.dao.BookDao;
import com.java2nb.books.domain.BookDO;
import com.java2nb.books.service.BookService;
@Service
public class BookServiceImpl implements BookService {
@Autowired
private BookDao bookDao;
@Override
public BookDO get(Long id){
return bookDao.get(id);
}
@Override
public List<BookDO> list(Map<String, Object> map){
return bookDao.list(map);
}
@Override
public int count(Map<String, Object> map){
return bookDao.count(map);
}
@Override
public int save(BookDO book){
return bookDao.save(book);
}
@Override
public int update(BookDO book){
return bookDao.update(book);
}
@Override
public int remove(Long id){
return bookDao.remove(id);
}
@Override
public int batchRemove(Long[] ids){
return bookDao.batchRemove(ids);
}
}

30
novel-admin/src/main/java/com/java2nb/books/util/RestTemplateUtil.java Normal file
View File

@ -0,0 +1,30 @@
package com.java2nb.books.util;
import org.springframework.http.client.HttpComponentsClientHttpRequestFactory;
import org.springframework.http.converter.HttpMessageConverter;
import org.springframework.http.converter.StringHttpMessageConverter;
import org.springframework.web.client.RestTemplate;
import java.nio.charset.Charset;
import java.util.List;
public class RestTemplateUtil {
public static RestTemplate getInstance(String charset) {
HttpComponentsClientHttpRequestFactory httpRequestFactory = new HttpComponentsClientHttpRequestFactory();
httpRequestFactory.setConnectionRequestTimeout(3000);
httpRequestFactory.setConnectTimeout(3000);
httpRequestFactory.setReadTimeout(10000);
RestTemplate restTemplate = new RestTemplate(httpRequestFactory);
List<HttpMessageConverter<?>> list = restTemplate.getMessageConverters();
for (HttpMessageConverter<?> httpMessageConverter : list) {
if(httpMessageConverter instanceof StringHttpMessageConverter) {
((StringHttpMessageConverter) httpMessageConverter).setDefaultCharset(Charset.forName(charset));
break;
}
}
return restTemplate;
}
}