201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-27 01:30:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

集成elasticsearch搜索引擎

Browse Source
This commit is contained in:
xiongxiaoyang 2020-05-20 18:08:29 +08:00
parent 4aa6b82143
commit 8a628f081f
14 changed files with 564 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
es/index_create.txt Normal file
View File

@ -0,0 +1,77 @@
PUT /novel
{
"mappings" : {
"book" : {
"properties" : {
"id" : {
"type" : "long"
},
"authorId" : {
"type" : "long"
},
"authorName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"bookName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"bookDesc" : {
"type" : "text",
"analyzer": "ik_smart"
},
"bookStatus" : {
"type" : "short"
},
"catId" : {
"type" : "integer"
},
"catName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"lastIndexId" : {
"type" : "long"
},
"lastIndexName" : {
"type" : "text",
"analyzer": "ik_smart"
},
"lastIndexUpdateTime" : {
"type": "keyword"
},
"picUrl" : {
"type" : "keyword"
},
"score" : {
"type" : "float"
},
"wordCount" : {
"type" : "integer"
},
"workDirection" : {
"type" : "short"
},
"visitCount" : {
"type": "long"
}
}
}
}
}

2
novel-admin/src/main/resources/static/js/plugins/switchery/switchery.js
View File

File diff suppressed because one or more lines are too long

4
novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java vendored
View File

@ -46,4 +46,8 @@ public interface CacheKey {
* */
String RUNNING_CRAWL_THREAD_KEY_PREFIX = "runningCrawlTreadDataKeyPrefix";
/**
* 上一次搜索引擎更新的时间
* */
String ES_LAST_UPDATE_TIME = "esLastUpdateTime";
}

73
novel-common/src/main/java/com/java2nb/novel/core/utils/StringUtil.java Normal file
View File

@ -0,0 +1,73 @@
package com.java2nb.novel.core.utils;
import org.apache.commons.lang3.StringUtils;
import java.util.Arrays;
/**
* @author xiongxiaoyang
*/
public class StringUtil {
/**
* 将驼峰式命名的字符串转换为下划线大写方式。如果转换前的驼峰式命名的字符串为空，则返回空字符串。</br>
* 例如：HelloWorld->HELLO_WORLD
* @param name 转换前的驼峰式命名的字符串
* @return 转换后下划线大写方式命名的字符串
*/
public static String underscoreName(String name) {
StringBuilder result = new StringBuilder();
if (name != null && name.length() > 0) {
// 将第一个字符处理成大写
result.append(name.substring(0, 1).toUpperCase());
// 循环处理其余字符
for (int i = 1; i < name.length(); i++) {
String s = name.substring(i, i + 1);
// 在大写字母前添加下划线
if (s.equals(s.toUpperCase()) && !Character.isDigit(s.charAt(0))) {
result.append("_");
}
// 其他字符直接转成大写
result.append(s.toUpperCase());
}
}
return result.toString();
}
/**
* 将下划线大写方式命名的字符串转换为驼峰式。如果转换前的下划线大写方式命名的字符串为空，则返回空字符串。</br>
* 例如：HELLO_WORLD->HelloWorld
* @param name 转换前的下划线大写方式命名的字符串
* @return 转换后的驼峰式命名的字符串
*/
public static String camelName(String name) {
StringBuilder result = new StringBuilder();
// 快速检查
if (name == null || name.isEmpty()) {
// 没必要转换
return "";
} else if (!name.contains("_")) {
// 不含下划线，仅将首字母小写
return name.substring(0, 1).toLowerCase() + name.substring(1);
}
// 用下划线将原始字符串分割
String camels[] = name.split("_");
for (String camel : camels) {
// 跳过原始字符串中开头、结尾的下换线或双重下划线
if (camel.isEmpty()) {
continue;
}
// 处理真正的驼峰片段
if (result.length() == 0) {
// 第一个驼峰片段，全部字母都小写
result.append(camel.toLowerCase());
} else {
// 其他的驼峰片段，首字母大写
result.append(camel.substring(0, 1).toUpperCase());
result.append(camel.substring(1).toLowerCase());
}
}
return result.toString();
}
}

12
novel-front/pom.xml
View File

@ -28,6 +28,18 @@
</dependency>
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>${jest.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>com.alipay.sdk</groupId>
<artifactId>alipay-sdk-java</artifactId>

4
novel-front/src/main/java/com/java2nb/novel/controller/BookController.java
View File

@ -79,8 +79,8 @@ public class BookController extends BaseController{
* */
@PostMapping("searchByPage")
public ResultBean searchByPage(BookSP bookSP, @RequestParam(value = "curr", defaultValue = "1") int page, @RequestParam(value = "limit", defaultValue = "20") int pageSize){
List<BookVO> books = bookService.searchByPage(bookSP,page,pageSize);
return ResultBean.ok(new PageInfo<>(books));
PageInfo<BookVO> pageInfo = bookService.searchByPage(bookSP,page,pageSize);
return ResultBean.ok(pageInfo);
}
/**

95
novel-front/src/main/java/com/java2nb/novel/core/schedule/BookToEsSchedule.java Normal file
View File

@ -0,0 +1,95 @@
package com.java2nb.novel.core.schedule;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.vo.EsBookVO;
import io.searchbox.client.JestClient;
import io.searchbox.core.DocumentResult;
import io.searchbox.core.Index;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
/**
* 小说导入搜索引擎
*
* @author Administrator
*/
@ConditionalOnProperty(prefix = "spring.elasticsearch", name = "enable", havingValue = "1")
@Service
@RequiredArgsConstructor
@Slf4j
public class BookToEsSchedule {
private final BookService bookService;
private final CacheService cacheService;
private final JestClient jestClient;
private boolean lock = false;
/**
* 5秒导入一次
*/
@Scheduled(fixedRate = 1000 * 5)
public void saveToEs() {
if (!lock) {
lock = true;
Date currentDate = new Date();
try {
//查询需要更新的小说
Date lastDate = (Date) cacheService.getObject(CacheKey.ES_LAST_UPDATE_TIME);
if (lastDate == null) {
lastDate = new SimpleDateFormat("yyyy-MM-dd").parse("2020-01-01");
}
long count ;
int page = 1;
do {
List<Book> books = bookService.queryBookByUpdateTimeByPage(lastDate, currentDate,page,100);
for(Book book : books) {
//导入到ES
EsBookVO esBookVO = new EsBookVO();
BeanUtils.copyProperties(book,esBookVO,"lastIndexUpdateTime");
esBookVO.setLastIndexUpdateTime(new SimpleDateFormat("yyyy/MM/dd HH:mm").format(book.getLastIndexUpdateTime()));
Index action = new Index.Builder(esBookVO).index("novel").type("book").id(book.getId().toString()).build();
jestClient.execute(action);
}
count = books.size();
page++;
}while (count == 100);
cacheService.setObject(CacheKey.ES_LAST_UPDATE_TIME, currentDate);
} catch (Exception e) {
log.error(e.getMessage(),e);
}
lock = false;
}
}
}

3
novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java
View File

@ -5,6 +5,7 @@ import com.java2nb.novel.search.BookSP;
import com.java2nb.novel.vo.BookVO;
import org.apache.ibatis.annotations.Param;
import java.util.Date;
import java.util.List;
/**
@ -15,7 +16,7 @@ public interface FrontBookMapper extends BookMapper {
List<BookVO> searchByPage(BookSP params);
void addVisitCount(@Param("bookId") Long bookId);
void addVisitCount(@Param("bookId") Long bookId, @Param("date") Date date);
List<Book> listRecBookByCatId(@Param("catId") Integer catId);

17
novel-front/src/main/java/com/java2nb/novel/service/BookService.java
View File

@ -1,12 +1,14 @@
package com.java2nb.novel.service;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.search.BookSP;
import com.java2nb.novel.vo.BookCommentVO;
import com.java2nb.novel.vo.BookSettingVO;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.vo.BookVO;
import java.util.Date;
import java.util.List;
import java.util.Map;
@ -44,9 +46,9 @@ public interface BookService {
* @param params 搜索参数
* @param page 页码
* @param pageSize 分页大小
* @return 小说集合
* @return 小说集合分页信息
* */
List<BookVO> searchByPage(BookSP params, int page, int pageSize);
PageInfo searchByPage(BookSP params, int page, int pageSize);
/**
* 查询小说分类列表
@ -224,4 +226,15 @@ public interface BookService {
* @param authorId 作者ID
* */
void addBookContent(Long bookId, String indexName, String content, Long authorId);
/**
* 根据更新时间分页查询书籍列表
* @param startDate 开始时间，包括该时间
* @param endDate 结束时间，不包括该时间
* @param page 页码
* @param pageSize 每页数量
* @return 书籍列表
* */
List<Book> queryBookByUpdateTimeByPage(Date startDate, Date endDate, int page, int pageSize);
}

197
novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -1,15 +1,14 @@
package com.java2nb.novel.service.impl;
import com.fasterxml.jackson.databind.JavaType;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.pagehelper.PageHelper;
import com.github.pagehelper.PageInfo;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.enums.ResponseStatus;
import com.java2nb.novel.core.exception.BusinessException;
import com.java2nb.novel.core.utils.BeanUtil;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.core.utils.IdWorker;
import com.java2nb.novel.core.utils.*;
import com.java2nb.novel.entity.*;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.mapper.*;
@ -19,9 +18,18 @@ import com.java2nb.novel.service.BookService;
import com.java2nb.novel.vo.BookCommentVO;
import com.java2nb.novel.vo.BookSettingVO;
import com.java2nb.novel.vo.BookVO;
import com.java2nb.novel.vo.EsBookVO;
import io.searchbox.client.JestClient;
import io.searchbox.core.*;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.mybatis.dynamic.sql.SortSpecification;
import org.mybatis.dynamic.sql.render.RenderingStrategies;
import org.mybatis.dynamic.sql.select.render.SelectStatementProvider;
@ -31,6 +39,7 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import tk.mybatis.orderbyhelper.OrderByHelper;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@ -50,6 +59,7 @@ import static org.mybatis.dynamic.sql.select.SelectDSL.select;
*/
@Service
@RequiredArgsConstructor
@Slf4j
public class BookServiceImpl implements BookService {
/**
@ -58,6 +68,9 @@ public class BookServiceImpl implements BookService {
@Value("${pic.save.path}")
private String picSavePath;
@Value("${spring.elasticsearch.enable}")
private Integer esEnable;
private final FrontBookSettingMapper bookSettingMapper;
private final FrontBookMapper bookMapper;
@ -76,6 +89,8 @@ public class BookServiceImpl implements BookService {
private final AuthorService authorService;
private final JestClient jestClient;
@SneakyThrows
@Override
@ -170,19 +185,163 @@ public class BookServiceImpl implements BookService {
return result;
}
@SneakyThrows
@Override
public List<BookVO> searchByPage(BookSP params, int page, int pageSize) {
PageHelper.startPage(page, pageSize);
public PageInfo searchByPage(BookSP params, int page, int pageSize) {
if (params.getUpdatePeriod() != null) {
long cur = System.currentTimeMillis();
long period = params.getUpdatePeriod() * 24 * 3600 * 1000;
long time = cur - period;
params.setUpdateTimeMin(new Date(time));
}
if (StringUtils.isNotBlank(params.getSort())) {
OrderByHelper.orderBy(params.getSort() + " desc");
if(esEnable == 1) {
List<EsBookVO> bookList = new ArrayList<>(0);
//使用搜索引擎搜索
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// 构造查询哪个字段
if (StringUtils.isNoneBlank(params.getKeyword())) {
boolQueryBuilder = boolQueryBuilder.must(QueryBuilders.queryStringQuery(params.getKeyword()));
}
// 作品方向
if (params.getWorkDirection() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("workDirection", params.getWorkDirection()));
}
// 分类
if (params.getCatId() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("catId", params.getCatId()));
}
if (params.getBookStatus() != null) {
boolQueryBuilder.filter(QueryBuilders.termQuery("bookStatus", params.getBookStatus()));
}
if(params.getWordCountMin() == null){
params.setWordCountMin(0);
}
if(params.getWordCountMax() == null){
params.setWordCountMax(Integer.MAX_VALUE);
}
boolQueryBuilder.filter(QueryBuilders.rangeQuery("wordCount").gte(params.getWordCountMin()).lte(params.getWordCountMax()));
if(params.getUpdateTimeMin() != null){
boolQueryBuilder.filter(QueryBuilders.rangeQuery("lastIndexUpdateTime").gte(params.getUpdateTimeMin()));
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
Count count = new Count.Builder().addIndex("novel").addType("book")
.query(searchSourceBuilder.toString()).build();
CountResult results = jestClient.execute(count);
Double total = results.getCount();
// 设置高亮字段
// 临时屏蔽小程序未处理的高亮字段，等小程序处理后再放开
HighlightBuilder highlightBuilder = new HighlightBuilder();
highlightBuilder.field("authorName");
highlightBuilder.field("bookName");
highlightBuilder.field("bookDesc");
highlightBuilder.field("lastIndexName");
highlightBuilder.field("catName");
highlightBuilder.preTags("<span style='color:red'>").postTags("</span>");
highlightBuilder.fragmentSize(200);
searchSourceBuilder.highlighter(highlightBuilder);
//设置排序
if(params.getSort() != null){
searchSourceBuilder.sort(StringUtil.camelName(params.getSort()), SortOrder.DESC);
}
// 设置分页
searchSourceBuilder.from((page - 1) * pageSize);
searchSourceBuilder.size(pageSize);
// 构建Search对象
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex("novel").addType("book").build();
log.debug(search.toString());
SearchResult result ;
result = jestClient.execute(search);
log.debug(result.getJsonString());
Map resultMap = new ObjectMapper().readValue(result.getJsonString(), Map.class);
if (resultMap.get("hits") != null) {
Map hitsMap = (Map) resultMap.get("hits");
if (hitsMap.size() > 0 && hitsMap.get("hits") != null) {
List hitsList = (List) hitsMap.get("hits");
if (hitsList.size() > 0 && result.getSourceAsString() != null) {
JavaType jt = new ObjectMapper().getTypeFactory().constructParametricType(ArrayList.class, EsBookVO.class);
bookList = new ObjectMapper().readValue("[" + result.getSourceAsString() + "]", jt);
if (bookList != null) {
for (int i = 0; i < bookList.size(); i++) {
hitsMap = (Map) hitsList.get(i);
Map highlightMap = (Map) hitsMap.get("highlight");
if (highlightMap != null && highlightMap.size() > 0) {
List<String> authorNameList = (List<String>) highlightMap.get("authorName");
if (authorNameList != null && authorNameList.size() > 0) {
bookList.get(i).setAuthorName(authorNameList.get(0));
}
List<String> bookNameList = (List<String>) highlightMap.get("bookName");
if (bookNameList != null && bookNameList.size() > 0) {
bookList.get(i).setBookName(bookNameList.get(0));
}
List<String> bookDescList = (List<String>) highlightMap.get("bookDesc");
if (bookDescList != null && bookDescList.size() > 0) {
bookList.get(i).setBookDesc(bookDescList.get(0));
}
List<String> lastIndexNameList = (List<String>) highlightMap.get("lastIndexName");
if (lastIndexNameList != null && lastIndexNameList.size() > 0) {
bookList.get(i).setLastIndexName(lastIndexNameList.get(0));
}
List<String> catNameList = (List<String>) highlightMap.get("catName");
if (catNameList != null && catNameList.size() > 0) {
bookList.get(i).setCatName(catNameList.get(0));
}
}
}
}
}
}
}
PageInfo<EsBookVO> pageInfo = new PageInfo<>(bookList);
pageInfo.setTotal(total.longValue());
pageInfo.setPageNum(page);
pageInfo.setPageSize(pageSize);
return pageInfo;
}else{
PageHelper.startPage(page, pageSize);
if (StringUtils.isNotBlank(params.getSort())) {
OrderByHelper.orderBy(params.getSort() + " desc");
}
return new PageInfo<>(bookMapper.searchByPage(params));
}
return bookMapper.searchByPage(params);
}
@Override
@ -197,7 +356,7 @@ public class BookServiceImpl implements BookService {
@Override
public Book queryBookDetail(Long bookId) {
SelectStatementProvider selectStatement = select(id, catName, catId, picUrl, bookName, authorId, authorName, bookDesc, bookStatus, visitCount, wordCount, lastIndexId, lastIndexName, lastIndexUpdateTime,score,status)
SelectStatementProvider selectStatement = select(book.allColumns())
.from(book)
.where(id, isEqualTo(bookId))
.build()
@ -316,8 +475,7 @@ public class BookServiceImpl implements BookService {
@Override
public void addVisitCount(Long bookId) {
bookMapper.addVisitCount(bookId);
bookMapper.addVisitCount(bookId,new Date());
}
@Override
@ -503,7 +661,7 @@ public class BookServiceImpl implements BookService {
.render(RenderingStrategies.MYBATIS3));
}
@Transactional
@Transactional(rollbackFor = Exception.class)
@Override
public void addBookContent(Long bookId, String indexName, String content, Long authorId) {
@ -556,5 +714,18 @@ public class BookServiceImpl implements BookService {
}
@Override
public List<Book> queryBookByUpdateTimeByPage(Date startDate, Date endDate, int page, int pageSize) {
PageHelper.startPage(page,pageSize);
return bookMapper.selectMany(select(book.allColumns())
.from(book)
.where(updateTime,isGreaterThanOrEqualTo(startDate))
.and(updateTime,isLessThan(endDate))
.build()
.render(RenderingStrategies.MYBATIS3));
}
}

85
novel-front/src/main/java/com/java2nb/novel/vo/EsBookVO.java Normal file
View File

@ -0,0 +1,85 @@
package com.java2nb.novel.vo;
import com.fasterxml.jackson.annotation.JsonFormat;
import com.java2nb.novel.entity.Book;
import lombok.Data;
import javax.annotation.Generated;
import java.io.Serializable;
import java.util.Date;
/**
* @author Administrator
*/
@Data
public class EsBookVO {
private Long id;
private Byte workDirection;
private Integer catId;
private String catName;
private String picUrl;
private String bookName;
private Long authorId;
private String authorName;
private String bookDesc;
private Float score;
private Byte bookStatus;
private Long visitCount;
private Integer wordCount;
private Integer commentCount;
private Long lastIndexId;
private String lastIndexName;
private String lastIndexUpdateTime;
private Byte isVip;
private Byte status;
private Integer crawlSourceId;
private String crawlBookId;
private Byte crawlIsStop;
}

9
novel-front/src/main/resources/application.yml
View File

@ -6,6 +6,15 @@ spring:
active: dev
include: alipay
elasticsearch:
#是否开启搜索引擎，1：开启，0：不开启
enable: 0
jest:
uris: http://198.245.61.51:9200
jwt:
secret: novel!#20191230
expiration: 604800

5
novel-front/src/main/resources/mybatis/mapping/BookMapper.xml
View File

@ -34,8 +34,9 @@
</select>
<update id="addVisitCount" parameterType="long">
update book set visit_count = visit_count + 1 where id = #{bookId}
<update id="addVisitCount" >
update book set visit_count = visit_count + 1 , update_time = #{date}
where id = #{bookId}
</update>
<select id="listRecBookByCatId" parameterType="int" resultType="com.java2nb.novel.entity.Book">

2
pom.xml
View File

@ -36,6 +36,8 @@
<orderbyhelper.version>1.0.2</orderbyhelper.version>
<commons-lang3.version>3.4</commons-lang3.version>
<jjwt.version>0.9.0</jjwt.version>
<elasticsearch.version>6.2.2</elasticsearch.version>
<jest.version>6.3.1</jest.version>
</properties>
<dependencyManagement>