Compare commits

..

5 Commits

Author SHA1 Message Date
773ce159f7 v5.2.2 发布 2025-07-17 21:14:24 +08:00
91e7d2712b refactor: 重构sort和order参数校验功能 2025-07-17 20:53:41 +08:00
3db8828384 fix: 修复sort和order参数的SQL注入漏洞 2025-07-17 19:03:58 +08:00
54bd194b98 feat(novel-crawl): 增加爬虫源采集章节数量监控功能
可以监测到爬虫源在当前环境下是否可用
2025-07-16 19:52:07 +08:00
3d41cf3ebb perf(novel-crawl): 优化爬虫源列表排序
按照更新时间倒序
2025-07-15 18:53:31 +08:00
48 changed files with 404 additions and 60 deletions

View File

@ -5,7 +5,7 @@
<groupId>com.java2nb</groupId>
<artifactId>novel-admin</artifactId>
<version>5.2.1</version>
<version>5.2.2</version>
<packaging>jar</packaging>
<name>novel-admin</name>

View File

@ -0,0 +1,13 @@
package com.java2nb.common.annotation;
import java.lang.annotation.*;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@Target(ElementType.PARAMETER)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface ValidateSortOrder {
}

View File

@ -0,0 +1,56 @@
package com.java2nb.common.aspect;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.common.utils.SortWhitelistUtil;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.stereotype.Component;
import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
import java.util.Arrays;
import java.util.Map;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@Aspect
@Component
@RequiredArgsConstructor
public class SortOrderValidationAspect {
/**
* 拦截所有的mapper方法
*/
@SneakyThrows
@Around("execution(* com.java2nb.*.dao.*Dao.*(..))")
public Object validateSortAndOrder(ProceedingJoinPoint joinPoint) {
Object[] args = joinPoint.getArgs();
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Method method = signature.getMethod();
Annotation[][] parameterAnnotations = method.getParameterAnnotations();
for (int i = 0; i < parameterAnnotations.length; i++) {
boolean hasAnnotation = Arrays.stream(parameterAnnotations[i])
.anyMatch(a -> a.annotationType().equals(ValidateSortOrder.class));
if (hasAnnotation && args[i] instanceof Map map) {
if (map.get("sort") instanceof String sortStr) {
map.put("sort", SortWhitelistUtil.sanitizeColumn(sortStr));
}
if (map.get("order") instanceof String orderStr) {
map.put("order", SortWhitelistUtil.sanitizeOrder(orderStr));
}
}
}
return joinPoint.proceed(args);
}
}

View File

@ -1,5 +1,6 @@
package com.java2nb.common.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.common.domain.DictDO;
import java.util.List;
@ -19,7 +20,7 @@ public interface DictDao {
DictDO get(Long id);
List<DictDO> list(Map<String, Object> map);
List<DictDO> list(@ValidateSortOrder Map<String, Object> map);
int count(Map<String, Object> map);

View File

@ -1,5 +1,6 @@
package com.java2nb.common.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.common.domain.FileDO;
import java.util.List;
@ -18,7 +19,7 @@ public interface FileDao {
FileDO get(Long id);
List<FileDO> list(Map<String,Object> map);
List<FileDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -3,6 +3,7 @@ package com.java2nb.common.dao;
import java.util.List;
import java.util.Map;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.common.domain.GenColumnsDO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
@ -19,7 +20,7 @@ public interface GenColumnsDao {
GenColumnsDO get(Long id);
List<GenColumnsDO> list(Map<String,Object> map);
List<GenColumnsDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,6 @@
package com.java2nb.common.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.common.domain.LogDO;
import java.util.List;
@ -18,7 +19,7 @@ public interface LogDao {
LogDO get(Long id);
List<LogDO> list(Map<String,Object> map);
List<LogDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -0,0 +1,28 @@
package com.java2nb.common.utils;
import lombok.experimental.UtilityClass;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@UtilityClass
public class SortWhitelistUtil {
private final Set<String> allowedColumns = new HashSet<>(
Arrays.asList("id", "name", "create_time", "update_time", "order_num", "last_index_update_time", "word_count",
"visit_count"));
private final Set<String> allowedOrders = new HashSet<>(Arrays.asList("asc", "desc"));
public String sanitizeColumn(String input) {
return allowedColumns.contains(input.toLowerCase()) ? input.toLowerCase() : "id";
}
public String sanitizeOrder(String input) {
return allowedOrders.contains(input.toLowerCase()) ? input.toLowerCase() : "asc";
}
}

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.AuthorCodeDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface AuthorCodeDao {
AuthorCodeDO get(Long id);
List<AuthorCodeDO> list(Map<String,Object> map);
List<AuthorCodeDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,6 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.AuthorDO;
import java.util.Date;
@ -19,7 +20,7 @@ public interface AuthorDao {
AuthorDO get(Long id);
List<AuthorDO> list(Map<String,Object> map);
List<AuthorDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.BookCommentDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface BookCommentDao {
BookCommentDO get(Long id);
List<BookCommentDO> list(Map<String,Object> map);
List<BookCommentDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.BookContentDO;
import org.apache.ibatis.annotations.Mapper;
@ -18,7 +20,7 @@ public interface BookContentDao {
BookContentDO get(Long id);
List<BookContentDO> list(Map<String, Object> map);
List<BookContentDO> list(@ValidateSortOrder Map<String, Object> map);
int count(Map<String, Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.BookDO;
import org.apache.ibatis.annotations.Mapper;
@ -19,7 +21,7 @@ public interface BookDao {
BookDO get(Long id);
List<BookDO> list(Map<String, Object> map);
List<BookDO> list(@ValidateSortOrder Map<String, Object> map);
int count(Map<String, Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.BookIndexDO;
import org.apache.ibatis.annotations.Mapper;
@ -18,7 +20,7 @@ public interface BookIndexDao {
BookIndexDO get(Long id);
List<BookIndexDO> list(Map<String, Object> map);
List<BookIndexDO> list(@ValidateSortOrder Map<String, Object> map);
int count(Map<String, Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.BookSettingDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface BookSettingDao {
BookSettingDO get(Long id);
List<BookSettingDO> list(Map<String,Object> map);
List<BookSettingDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.CategoryDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface CategoryDao {
CategoryDO get(Integer id);
List<CategoryDO> list(Map<String,Object> map);
List<CategoryDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.FriendLinkDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface FriendLinkDao {
FriendLinkDO get(Integer id);
List<FriendLinkDO> list(Map<String,Object> map);
List<FriendLinkDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.NewsDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface NewsDao {
NewsDO get(Long id);
List<NewsDO> list(Map<String,Object> map);
List<NewsDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.PayDO;
import java.util.Date;
@ -19,7 +21,7 @@ public interface PayDao {
PayDO get(Long id);
List<PayDO> list(Map<String,Object> map);
List<PayDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.UserDO;
import org.apache.ibatis.annotations.Mapper;
@ -17,7 +19,7 @@ public interface UserDao {
UserDO get(Long id);
List<UserDO> list(Map<String, Object> map);
List<UserDO> list(@ValidateSortOrder Map<String, Object> map);
int count(Map<String, Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.UserFeedbackDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface UserFeedbackDao {
UserFeedbackDO get(Long id);
List<UserFeedbackDO> list(Map<String,Object> map);
List<UserFeedbackDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.novel.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.novel.domain.WebsiteInfoDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface WebsiteInfoDao {
WebsiteInfoDO get(Long id);
List<WebsiteInfoDO> list(Map<String,Object> map);
List<WebsiteInfoDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.DataPermDO;
import java.util.List;
@ -19,7 +21,7 @@ public interface DataPermDao {
DataPermDO get(Long id);
List<DataPermDO> list(Map<String,Object> map);
List<DataPermDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.DeptDO;
import java.util.List;
@ -19,7 +21,7 @@ public interface DeptDao {
DeptDO get(Long deptId);
List<DeptDO> list(Map<String,Object> map);
List<DeptDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.MenuDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface MenuDao {
MenuDO get(Long menuId);
List<MenuDO> list(Map<String,Object> map);
List<MenuDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.RoleDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface RoleDao {
RoleDO get(Long roleId);
List<RoleDO> list(Map<String,Object> map);
List<RoleDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.RoleDataPermDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface RoleDataPermDao {
RoleDataPermDO get(Long id);
List<RoleDataPermDO> list(Map<String,Object> map);
List<RoleDataPermDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.RoleMenuDO;
import java.util.List;
@ -18,7 +20,7 @@ public interface RoleMenuDao {
RoleMenuDO get(Long id);
List<RoleMenuDO> list(Map<String,Object> map);
List<RoleMenuDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.UserDO;
import java.util.List;
@ -19,7 +21,7 @@ public interface SysUserDao {
UserDO get(Long userId);
List<UserDO> list(Map<String,Object> map);
List<UserDO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -1,5 +1,7 @@
package com.java2nb.system.dao;
import com.java2nb.common.annotation.ValidateSortOrder;
import com.java2nb.system.domain.UserRoleDO;
import java.util.List;
@ -19,7 +21,7 @@ public interface UserRoleDao {
UserRoleDO get(Long id);
List<UserRoleDO> list(Map<String, Object> map);
List<UserRoleDO> list(@ValidateSortOrder Map<String, Object> map);
int count(Map<String, Object> map);

View File

@ -18,7 +18,7 @@ public interface ${className}Dao {
${className}DO get(${pk.javaType} ${pk.attrname});
List<${className}DO> list(Map<String,Object> map);
List<${className}DO> list(@ValidateSortOrder Map<String,Object> map);
int count(Map<String,Object> map);

View File

@ -44,7 +44,7 @@ public interface ${className}Mapper {
"limit #{offset}, #{limit}" +
"</if>"+
"</script>")
List<${className}DO> list(Map<String,Object> map);
List<${className}DO> list(@ValidateSortOrder Map<String,Object> map);
@Select("<script>" +
"select count(*) from ${tableName} " +

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>5.2.1</version>
<version>5.2.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>
@ -22,6 +22,12 @@
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!-- Aop 相关 -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-aop</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-thymeleaf</artifactId>

View File

@ -0,0 +1,13 @@
package com.java2nb.novel.core.annotation;
import java.lang.annotation.*;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@Target(ElementType.PARAMETER)
@Retention(RetentionPolicy.RUNTIME)
@Documented
public @interface ValidateSortOrder {
}

View File

@ -0,0 +1,101 @@
package com.java2nb.novel.core.aspect;
import com.java2nb.novel.core.annotation.ValidateSortOrder;
import com.java2nb.novel.core.utils.SortWhitelistUtil;
import com.java2nb.novel.core.vo.SortOrderVO;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.stereotype.Component;
import java.lang.annotation.Annotation;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.util.*;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@Aspect
@Component
@RequiredArgsConstructor
public class SortOrderValidationAspect {
/**
* 拦截所有的mapper方法
*/
@SneakyThrows
@Around("execution(* com.java2nb.novel.mapper.*Mapper.*(..))")
public Object processSortOrderFields(ProceedingJoinPoint joinPoint) {
Object[] args = joinPoint.getArgs();
MethodSignature signature = (MethodSignature) joinPoint.getSignature();
Method method = signature.getMethod();
Annotation[][] parameterAnnotations = method.getParameterAnnotations();
for (int i = 0; i < parameterAnnotations.length; i++) {
boolean hasAnnotation = Arrays.stream(parameterAnnotations[i])
.anyMatch(a -> a.annotationType().equals(ValidateSortOrder.class));
if (hasAnnotation && args[i] != null) {
handleAnnotatedParameter(args[i]);
}
}
return joinPoint.proceed(args);
}
@SneakyThrows
private void handleAnnotatedParameter(Object obj) {
if (obj instanceof SortOrderVO sortOrderVO){
processSortOrderVO(sortOrderVO);
} else if (obj instanceof Map<?, ?> map) {
processMap(map);
} else {
processGenericObject(obj);
}
}
private void processSortOrderVO(SortOrderVO sortOrderVO) {
if(sortOrderVO.getSort() != null){
sortOrderVO.setSort(SortWhitelistUtil.sanitizeColumn(sortOrderVO.getSort()));
}
if(sortOrderVO.getOrder() != null){
sortOrderVO.setOrder(SortWhitelistUtil.sanitizeOrder(sortOrderVO.getOrder()));
}
}
private void processMap(Map map) {
if (map.get("sort") instanceof String sortStr) {
map.put("sort", SortWhitelistUtil.sanitizeColumn(sortStr));
}
if (map.get("order") instanceof String orderStr) {
map.put("order", SortWhitelistUtil.sanitizeOrder(orderStr));
}
}
@SneakyThrows
private void processGenericObject(Object obj) {
for (Field field : obj.getClass().getDeclaredFields()) {
switch (field.getName()) {
case "sort", "order" -> {
field.setAccessible(true);
Object value = field.get(obj);
if (value instanceof String strValue) {
String sanitized = "sort".equals(field.getName())
? SortWhitelistUtil.sanitizeColumn(strValue)
: SortWhitelistUtil.sanitizeOrder(strValue);
field.set(obj, sanitized);
}
}
default -> {
}
}
}
}
}

View File

@ -0,0 +1,28 @@
package com.java2nb.novel.core.utils;
import lombok.experimental.UtilityClass;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@UtilityClass
public class SortWhitelistUtil {
private final Set<String> allowedColumns = new HashSet<>(
Arrays.asList("id", "name", "create_time", "update_time", "last_index_update_time", "word_count",
"visit_count"));
private final Set<String> allowedOrders = new HashSet<>(Arrays.asList("asc", "desc"));
public String sanitizeColumn(String input) {
return allowedColumns.contains(input.toLowerCase()) ? input.toLowerCase() : "id";
}
public String sanitizeOrder(String input) {
return allowedOrders.contains(input.toLowerCase()) ? input.toLowerCase() : "asc";
}
}

View File

@ -0,0 +1,16 @@
package com.java2nb.novel.core.vo;
import lombok.Data;
/**
* @author xiongxiaoyang
* @date 2025/7/17
*/
@Data
public class SortOrderVO {
private String sort;
private String order;
}

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>5.2.1</version>
<version>5.2.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -12,6 +12,8 @@ import io.github.xxyopen.util.IdWorker;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.text.ParseException;
@ -34,6 +36,13 @@ public class CrawlParser {
private final CrawlHttpClient crawlHttpClient;
private final StringRedisTemplate stringRedisTemplate;
/**
* 爬虫源采集章节数量缓存key
*/
private static final String CRAWL_SOURCE_CHAPTER_COUNT_CACHE_KEY = "crawlSource:chapterCount:";
/**
* 爬虫任务进度
*/
@ -53,6 +62,20 @@ public class CrawlParser {
crawlTaskProgress.remove(taskId);
}
/**
* 获取爬虫源采集的章节数量
*/
public Long getCrawlSourceChapterCount(Integer sourceId) {
return Optional.ofNullable(
stringRedisTemplate.opsForValue().get(CRAWL_SOURCE_CHAPTER_COUNT_CACHE_KEY + sourceId)).map(v -> {
try {
return Long.parseLong(v);
} catch (NumberFormatException e) {
return 0L;
}
}).orElse(0L);
}
public void parseBook(RuleBean ruleBean, String bookId, CrawlBookHandler handler)
throws InterruptedException {
Book book = new Book();
@ -182,7 +205,7 @@ public class CrawlParser {
handler.handle(book);
}
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean,
public boolean parseBookIndexAndContent(String sourceBookId, Book book, RuleBean ruleBean, Integer sourceId,
Map<Integer, BookIndex> existBookIndexMap, CrawlBookChapterHandler handler, CrawlSingleTask task)
throws InterruptedException {
@ -314,10 +337,12 @@ public class CrawlParser {
bookIndex.setUpdateTime(currentDate);
if (task != null) {
// 更新采集进度
// 更新单本任务采集进度
crawlTaskProgress.put(task.getId(), indexList.size());
}
// 更新爬虫源采集章节数量
stringRedisTemplate.opsForValue().increment(CRAWL_SOURCE_CHAPTER_COUNT_CACHE_KEY + sourceId);
}

View File

@ -74,7 +74,7 @@ public class StarterListener implements ServletContextInitializer {
needUpdateBook.getId());
//解析章节目录
crawlParser.parseBookIndexAndContent(needUpdateBook.getCrawlBookId(), book,
ruleBean, existBookIndexMap,
ruleBean, needUpdateBook.getCrawlSourceId(), existBookIndexMap,
chapter -> bookService.updateBookAndIndexAndContent(book,
chapter.getBookIndexList(),
chapter.getBookContentList(), existBookIndexMap), null);

View File

@ -100,14 +100,19 @@ public class CrawlServiceImpl implements CrawlService {
PageHelper.startPage(page, pageSize);
SelectStatementProvider render = select(id, sourceName, sourceStatus, createTime, updateTime)
.from(crawlSource)
.orderBy(updateTime)
.orderBy(updateTime.descending())
.build()
.render(RenderingStrategies.MYBATIS3);
List<CrawlSource> crawlSources = crawlSourceMapper.selectMany(render);
crawlSources.forEach(crawlSource -> crawlSource.setSourceStatus(
Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0)));
PageBean<CrawlSource> pageBean = PageBuilder.build(crawlSources);
pageBean.setList(BeanUtil.copyList(crawlSources, CrawlSourceVO.class));
List<CrawlSourceVO> crawlSourceVOS = BeanUtil.copyList(crawlSources, CrawlSourceVO.class);
crawlSourceVOS.forEach(crawlSource -> {
crawlSource.setSourceStatus(
Optional.ofNullable(crawlSourceStatusMap.get(crawlSource.getId())).orElse((byte) 0));
crawlSource.setChapterCount(crawlParser.getCrawlSourceChapterCount(crawlSource.getId()));
}
);
pageBean.setList(crawlSourceVOS);
return pageBean;
}
@ -386,7 +391,7 @@ public class CrawlServiceImpl implements CrawlService {
book.setCrawlLastTime(new Date());
book.setId(idWorker.nextId());
//解析章节目录
boolean parseIndexContentResult = crawlParser.parseBookIndexAndContent(bookId, book, ruleBean,
boolean parseIndexContentResult = crawlParser.parseBookIndexAndContent(bookId, book, ruleBean, sourceId,
new HashMap<>(0), chapter -> {
bookService.saveBookAndIndexAndContent(book, chapter.getBookIndexList(),
chapter.getBookContentList());

View File

@ -20,7 +20,7 @@ public class CrawlSourceVO extends CrawlSource{
@JsonFormat(timezone = "GMT+8", pattern = "yyyy-MM-dd HH:mm")
private Date updateTime;
private Long chapterCount;
@Override
public String toString() {

View File

@ -43,7 +43,7 @@
<th class="style">
序号
</th>
<th class="chapter">
<th class="name">
爬虫源
</th>
<th class="name">
@ -52,6 +52,9 @@
<th class="name">
更新时间
</th>
<th class="goread">
采集数量
</th>
<th class="goread">
状态
</th>
@ -111,11 +114,17 @@
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
search(1, 10);
let curr = 1;
let limit = 10;
search();
setInterval(function(){
search();
}, 10000);
var pageCrawlSourceList = null;
function search(curr, limit) {
function search() {
$.ajax({
type: "get",
@ -134,13 +143,15 @@
" <td class=\"style bookclass\">\n" +
" [" + (i + 1) + "]\n" +
" </td>\n" +
" <td class=\"chapter\">\n" +
" <td class=\"name\">\n" +
" " + crawlSource.sourceName + "</td>\n" +
" <td class=\"name\" valsc=\"291|2037554|1\">"
+ crawlSource.createTime + "</td>\n" +
" <td class=\"name\">\n" +
" " + crawlSource.updateTime + "\n" +
" </td>\n" +
" <td class=\"goread\">\n" +
" " + crawlSource.chapterCount + "章</td>\n" +
" <td class=\"goread\" id='sourceStatus" + crawlSource.id + "'>" + (crawlSource.sourceStatus == 0 ? '停止运行' : '正在运行') +
" </td>\n" +
@ -169,7 +180,9 @@
//首次不执行
if (!first) {
search(obj.curr, obj.limit);
curr = obj.curr;
limit = obj.limit;
search();
} else {
}
@ -216,11 +229,11 @@
if (status == 0) {
//开启
$("#sourceStatus" + sourceId).html("正在运行");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 1 + ")'>关闭</a>");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 1 + ")'>关闭 </a>"+"<a href='javascript:updateCrawlSource(" + sourceId + ")'>修改 </a>");
} else {
//关闭
$("#sourceStatus" + sourceId).html("停止运行");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 0 + ")'>开启</a>");
$("#opt" + sourceId).html("<a href='javascript:openOrStopCrawl(" + sourceId + "," + 0 + ")'>开启 </a>"+"<a href='javascript:updateCrawlSource(" + sourceId + ")'>修改 </a>");
}

View File

@ -5,7 +5,7 @@
<parent>
<artifactId>novel</artifactId>
<groupId>com.java2nb</groupId>
<version>5.2.1</version>
<version>5.2.2</version>
</parent>
<modelVersion>4.0.0</modelVersion>

View File

@ -1,5 +1,6 @@
package com.java2nb.novel.mapper;
import com.java2nb.novel.core.annotation.ValidateSortOrder;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.vo.BookSpVO;
import com.java2nb.novel.vo.BookVO;
@ -13,7 +14,7 @@ import java.util.List;
public interface FrontBookMapper extends BookMapper {
List<BookVO> searchByPage(BookSpVO params);
List<BookVO> searchByPage(@ValidateSortOrder BookSpVO params);
void addVisitCount(@Param("bookId") Long bookId, @Param("visitCount") Integer visitCount);

View File

@ -1,12 +1,7 @@
package com.java2nb.novel.mapper;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.vo.BookSpVO;
import com.java2nb.novel.vo.BookVO;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* @author Administrator
*/

View File

@ -1,5 +1,6 @@
package com.java2nb.novel.vo;
import com.java2nb.novel.core.vo.SortOrderVO;
import lombok.Data;
import java.util.Date;
@ -9,7 +10,7 @@ import java.util.Date;
* @author 11797
*/
@Data
public class BookSpVO {
public class BookSpVO extends SortOrderVO {
private String keyword;
@ -29,9 +30,5 @@ public class BookSpVO {
private Long updatePeriod;
private String sort;
}

View File

@ -5,7 +5,7 @@
<groupId>com.java2nb</groupId>
<artifactId>novel</artifactId>
<version>5.2.1</version>
<version>5.2.2</version>
<modules>
<module>novel-common</module>
<module>novel-front</module>