201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-26 09:20:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #71 from a100488/task_updateCrawlSource

Browse Source 
feat(新增编辑规则测试规则): 新增编辑规则测试规则
This commit is contained in:
201206030 2021-12-24 15:46:00 +08:00 committed by GitHub
commit bb2d95ad75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 1381 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
novel-admin/pom.xml
View File

@ -224,17 +224,17 @@
<!--<scope>provided</scope>-->
<!--</dependency>-->
</dependencies>
<build>
<!-- <build>
<plugins>
<!--<plugin>
&lt;!&ndash;<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<executable>true</executable>
</configuration>
</plugin>-->
<!--SpringBoot项目默认使用spring-boot-maven-plugin，要打成被其他项目引用的jar包，需要更换此插件-->
<!-- <plugin>
</plugin>&ndash;&gt;
&lt;!&ndash;SpringBoot项目默认使用spring-boot-maven-plugin，要打成被其他项目引用的jar包，需要更换此插件&ndash;&gt;
&lt;!&ndash; <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
@ -242,12 +242,91 @@
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>-->
</plugin>&ndash;&gt;
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>-->
<build>
<plugins>
<plugin>
<!--打包时去除第三方依赖-->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<layout>ZIP</layout>
<includes>
<include>
<groupId>non-exists</groupId>
<artifactId>non-exists</artifactId>
</include>
</includes>
</configuration>
</plugin>
<!--拷贝第三方依赖文件到指定目录-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<!--target/lib是依赖jar包的输出目录，根据自己喜好配置-->
<outputDirectory>target/lib</outputDirectory>
<excludeTransitive>false</excludeTransitive>
<stripVersion>false</stripVersion>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<!-- 文件夹 -->
<copy todir="${project.build.directory}/build/conf" overwrite="true">
<fileset dir="${basedir}/src/main/resources">
<include name="**/*.*"/>
<exclude name="mybatis/*/*.*"/>
</fileset>
</copy>
<move todir="${project.build.directory}/build/lib">
<fileset dir="target/lib"/>
</move>
<copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar"
tofile="${project.build.directory}/build/${project.artifactId}.jar" />
<fixcrlf srcdir="${basedir}/src/main/build/scripts" eol="unix"/>
<copy todir="${project.build.directory}/build/bin">
<fileset dir="${basedir}/src/main/build/scripts">
<include name="*.sh" />
<include name="*.txt" />
<include name="*.bat" />
</fileset>
</copy>
<zip destfile='${project.build.directory}/build/${project.artifactId}.zip'>
<zipfileset filemode="755" dir= '${project.build.directory}/build/' />
</zip>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
<!--<distributionManagement>
<repository>

8
novel-admin/src/main/build/scripts/readme.txt Normal file
View File

@ -0,0 +1,8 @@
1：linux启动环境
sh start.sh
3：windows启动环境
windows-start.bat
3：linux停止应用
sh stop.sh

47
novel-admin/src/main/build/scripts/start.sh Normal file
View File

@ -0,0 +1,47 @@
#!/bin/bash
ENGINE=novel-admin.jar
cd ../
#部署目路
DEPLOY_DIR=`pwd`
#获取到当前目录的名称
SERVER_NAME=`basename $DEPLOY_DIR`
#应用进程
PIDS=`ps -ef | grep java | grep "$ENGINE" |awk '{print $2}'`
#设置日志文件的输出目录
LOGS_DIR=$DEPLOY_DIR/logs
if [ ! -d $LOGS_DIR ]; then
mkdir $LOGS_DIR
fi
#日志
STDOUT_FILE=$LOGS_DIR/stdout.log
#JAVA 环境配置
JAVA_OPTS=" -Djava.net.preferIPv4Stack=true -Dlog.home=$LOGS_DIR"
JAVA_MEM_OPTS=" -server -Xms1024m -Xmx1024m -XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=256m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=50 -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCDateStamps -verbose:gc -XX:+PrintGCDetails -XX:+PrintHeapAtGC -Xloggc:$LOGS_DIR/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof "
#退出标志
RETVAL="0"
if [ -n "$PIDS" ]; then
echo "ERROR: The $SERVER_NAME already started!"
echo "PID: $PIDS"
exit $RETVAL
fi
nohup java -jar $JAVA_OPTS $JAVA_MEM_OPTS -Dloader.path=conf,lib $ENGINE > $STDOUT_FILE 2>&1 &
COUNT=0
while [ $COUNT -lt 1 ]; do
echo -e ".\c"
sleep 1
COUNT=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}' | wc -l`
if [ $COUNT -gt 0 ]; then
break
fi
done
echo "OK!"
PIDS=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}'`
echo "PID: $PIDS"
echo "STDOUT: $STDOUT_FILE"

33
novel-admin/src/main/build/scripts/stop.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
SERVER_NAME=novel-admin.jar
#应用进程
PIDS=`ps -ef | grep java | grep "$SERVER_NAME" |awk '{print $2}'`
if [ -z "$PIDS" ]; then
echo "ERROR: The $SERVER_NAME does not started!"
exit 1
fi
echo -e "Stopping the $SERVER_NAME ...\c"
for PID in $PIDS ; do
kill $PID > /dev/null 2>&1
done
COUNT=0
while [ $COUNT -lt 1 ]; do
echo -e ".\c"
sleep 1
COUNT=1
for PID in $PIDS ; do
PID_EXIST=`ps -f -p $PID | grep java`
if [ -n "$PID_EXIST" ]; then
COUNT=0
break
fi
done
done
echo "OK!"
echo "PID: $PIDS"
PIDS=""

10
novel-admin/src/main/build/scripts/windows-start.bat Normal file
View File

@ -0,0 +1,10 @@
@echo off
setlocal enabledelayedexpansion
set JAVA=java
set OPTS=-XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xms1024m -Xmx1024m -Xmn256m -Xss256k -XX:SurvivorRatio=8 -XX:+UseConcMarkSweepGC
set ENGINE=novel-admin.jar
cd ../
java -jar %OPTS% -Dloader.path=conf,lib %ENGINE%
pause

2
novel-admin/src/main/resources/logback-spring.xml
View File

@ -56,7 +56,7 @@
<!-- 指定项目中某个包，当有日志操作行为时的日志记录级别 -->
<!-- com.maijinjie.springboot 为根包，也就是只要是发生在这个根包下面的所有日志操作行为的权限都是DEBUG -->
<!-- 级别依次为【从高到低】：FATAL > ERROR > WARN > INFO > DEBUG > TRACE -->
<logger name="com.java2nb" level="DEBUG">
<logger name="com.java2nb" level="DEBUG" additivity="false">
<appender-ref ref="debug" />
</logger>
</configuration>

6
novel-common/src/main/java/com/java2nb/novel/core/cache/CacheKey.java vendored
View File

@ -65,4 +65,8 @@ public interface CacheKey {
* 累积的小说点击量
* */
String BOOK_ADD_VISIT_COUNT = "bookAddVisitCount";
}
/**
* 测试爬虫规则缓存
*/
String BOOK_TEST_PARSE = "testParse";
}

6
novel-common/src/main/java/com/java2nb/novel/entity/News.java
View File

@ -1,9 +1,11 @@
package com.java2nb.novel.entity;
import java.io.Serializable;
import java.util.Date;
import javax.annotation.Generated;
public class News {
public class News implements Serializable {
@Generated("org.mybatis.generator.api.MyBatisGenerator")
private Long id;
@ -146,4 +148,4 @@ public class News {
public void setContent(String content) {
this.content = content == null ? null : content.trim();
}
}
}

73
novel-crawl/pom.xml
View File

@ -32,11 +32,82 @@
<build>
<plugins>
<plugin>
<!--打包时去除第三方依赖-->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<layout>ZIP</layout>
<includes>
<include>
<groupId>non-exists</groupId>
<artifactId>non-exists</artifactId>
</include>
</includes>
</configuration>
</plugin>
<!--拷贝第三方依赖文件到指定目录-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<!--target/lib是依赖jar包的输出目录，根据自己喜好配置-->
<outputDirectory>target/lib</outputDirectory>
<excludeTransitive>false</excludeTransitive>
<stripVersion>false</stripVersion>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<!-- 文件夹 -->
<copy todir="${project.build.directory}/build/conf" overwrite="true">
<fileset dir="${basedir}/src/main/resources">
<include name="**/*.*"/>
<exclude name="mybatis/*/*.*"/>
</fileset>
</copy>
<move todir="${project.build.directory}/build/lib">
<fileset dir="target/lib"/>
</move>
<copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar"
tofile="${project.build.directory}/build/${project.artifactId}.jar" />
<fixcrlf srcdir="${basedir}/src/main/build/scripts" eol="unix"/>
<copy todir="${project.build.directory}/build/bin">
<fileset dir="${basedir}/src/main/build/scripts">
<include name="*.sh" />
<include name="*.txt" />
<include name="*.bat" />
</fileset>
</copy>
<zip destfile='${project.build.directory}/build/${project.artifactId}.zip'>
<zipfileset filemode="755" dir= '${project.build.directory}/build/' />
</zip>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
</project>

8
novel-crawl/src/main/build/scripts/readme.txt Normal file
View File

@ -0,0 +1,8 @@
1：linux启动环境
sh start.sh
3：windows启动环境
windows-start.bat
3：linux停止应用
sh stop.sh

47
novel-crawl/src/main/build/scripts/start.sh Normal file
View File

@ -0,0 +1,47 @@
#!/bin/bash
ENGINE=novel-crawl.jar
cd ../
#部署目路
DEPLOY_DIR=`pwd`
#获取到当前目录的名称
SERVER_NAME=`basename $DEPLOY_DIR`
#应用进程
PIDS=`ps -ef | grep java | grep "$ENGINE" |awk '{print $2}'`
#设置日志文件的输出目录
LOGS_DIR=$DEPLOY_DIR/logs
if [ ! -d $LOGS_DIR ]; then
mkdir $LOGS_DIR
fi
#日志
STDOUT_FILE=$LOGS_DIR/stdout.log
#JAVA 环境配置
JAVA_OPTS=" -Djava.net.preferIPv4Stack=true -Dlog.home=$LOGS_DIR"
JAVA_MEM_OPTS=" -server -Xms1024m -Xmx1024m -XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=256m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=50 -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCDateStamps -verbose:gc -XX:+PrintGCDetails -XX:+PrintHeapAtGC -Xloggc:$LOGS_DIR/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof "
#退出标志
RETVAL="0"
if [ -n "$PIDS" ]; then
echo "ERROR: The $SERVER_NAME already started!"
echo "PID: $PIDS"
exit $RETVAL
fi
nohup java -jar $JAVA_OPTS $JAVA_MEM_OPTS -Dloader.path=conf,lib $ENGINE > $STDOUT_FILE 2>&1 &
COUNT=0
while [ $COUNT -lt 1 ]; do
echo -e ".\c"
sleep 1
COUNT=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}' | wc -l`
if [ $COUNT -gt 0 ]; then
break
fi
done
echo "OK!"
PIDS=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}'`
echo "PID: $PIDS"
echo "STDOUT: $STDOUT_FILE"

33
novel-crawl/src/main/build/scripts/stop.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
SERVER_NAME=novel-crawl.jar
#应用进程
PIDS=`ps -ef | grep java | grep "$SERVER_NAME" |awk '{print $2}'`
if [ -z "$PIDS" ]; then
echo "ERROR: The $SERVER_NAME does not started!"
exit 1
fi
echo -e "Stopping the $SERVER_NAME ...\c"
for PID in $PIDS ; do
kill $PID > /dev/null 2>&1
done
COUNT=0
while [ $COUNT -lt 1 ]; do
echo -e ".\c"
sleep 1
COUNT=1
for PID in $PIDS ; do
PID_EXIST=`ps -f -p $PID | grep java`
if [ -n "$PID_EXIST" ]; then
COUNT=0
break
fi
done
done
echo "OK!"
echo "PID: $PIDS"
PIDS=""

10
novel-crawl/src/main/build/scripts/windows-start.bat Normal file
View File

@ -0,0 +1,10 @@
@echo off
setlocal enabledelayedexpansion
set JAVA=java
set OPTS=-XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xms1024m -Xmx1024m -Xmn256m -Xss256k -XX:SurvivorRatio=8 -XX:+UseConcMarkSweepGC
set ENGINE=novel-crawl.jar
cd ../
java -jar %OPTS% -Dloader.path=conf,lib %ENGINE%
pause

78
novel-crawl/src/main/java/com/java2nb/novel/controller/CrawlController.java
View File

@ -1,13 +1,26 @@
package com.java2nb.novel.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.cache.CacheKey;
import com.java2nb.novel.core.cache.CacheService;
import com.java2nb.novel.core.crawl.CrawlParser;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.core.utils.HttpUtil;
import com.java2nb.novel.entity.BookIndex;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
import com.java2nb.novel.service.CrawlService;
import com.java2nb.novel.utils.Constants;
import lombok.RequiredArgsConstructor;
import org.springframework.web.bind.annotation.*;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* @author Administrator
*/
@ -18,7 +31,7 @@ public class CrawlController {
private final CrawlService crawlService;
private final CacheService cacheService;
/**
* 新增爬虫源
* */
@ -38,7 +51,70 @@ public class CrawlController {
return ResultBean.ok(crawlService.listCrawlByPage(page,pageSize));
}
/**
* 获取爬虫源
* */
@GetMapping("getCrawlSource/{id}")
public ResultBean<CrawlSource> getCrawlSource(@PathVariable("id") Integer id){
CrawlSource crawlSource= crawlService.getCrawlSource(id);
return ResultBean.ok(crawlSource);
}
/**
* 测试规则
* @param rule
* @param url
* @param isRefresh
* @return
*/
@PostMapping("testParse")
public ResultBean<Object> testParse(String rule,String url,String isRefresh){
Map<String,Object> resultMap=new HashMap<>();
String html =null;
if(url.startsWith("https://")||url.startsWith("http://")){
String refreshCache="1";
if(!refreshCache.equals(isRefresh)) {
Object cache = cacheService.getObject(CacheKey.BOOK_TEST_PARSE + url);
if (cache == null) {
isRefresh="1";
}else {
html = (String) cache;
}
}
if(refreshCache.equals(isRefresh)){
html = HttpUtil.getByHttpClientWithChrome(url);
if (html != null) {
cacheService.setObject(CacheKey.BOOK_TEST_PARSE + url, html, 60 * 10);
}else{
resultMap.put("msg","html is null");
return ResultBean.ok(resultMap);
}
}
}else{
resultMap.put("html","url is null");
return ResultBean.ok(resultMap);
}
Pattern pattern = Pattern.compile(rule);
Matcher matcher = pattern.matcher(html);
boolean isFind = matcher.find();
resultMap.put("是否匹配",isFind);
if(isFind){
resultMap.put("匹配结果",matcher.group(1));
}
// resultMap.put("url",url);
return ResultBean.ok(resultMap);
}
/**
* 修改爬虫源
* */
@PostMapping("updateCrawlSource")
public ResultBean<Void> updateCrawlSource(CrawlSource source){
crawlService.updateCrawlSource(source);
return ResultBean.ok();
}
/**
* 开启或停止爬虫
* */

14
novel-crawl/src/main/java/com/java2nb/novel/service/CrawlService.java
View File

@ -1,6 +1,7 @@
package com.java2nb.novel.service;
import com.java2nb.novel.core.bean.PageBean;
import com.java2nb.novel.core.bean.ResultBean;
import com.java2nb.novel.core.crawl.RuleBean;
import com.java2nb.novel.entity.CrawlSingleTask;
import com.java2nb.novel.entity.CrawlSource;
@ -18,7 +19,11 @@ public interface CrawlService {
* */
void addCrawlSource(CrawlSource source);
/**
* 修改爬虫源
* @param source
*/
void updateCrawlSource(CrawlSource source);
/**
* 爬虫源分页列表
* @param page 当前页码
@ -106,4 +111,11 @@ public interface CrawlService {
* @param status 采集状态
* */
void updateCrawlSingleTask(CrawlSingleTask task, Byte status);
/**
* 获取采集规则详细
* @param id
* @return
*/
CrawlSource getCrawlSource(Integer id);
}

39
novel-crawl/src/main/java/com/java2nb/novel/service/impl/CrawlServiceImpl.java
View File

@ -39,6 +39,7 @@ import java.util.regex.Pattern;
import static com.java2nb.novel.core.utils.HttpUtil.getByHttpClientWithChrome;
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.*;
import static com.java2nb.novel.mapper.CrawlSourceDynamicSqlSupport.id;
import static org.mybatis.dynamic.sql.SqlBuilder.isEqualTo;
import static org.mybatis.dynamic.sql.select.SelectDSL.select;
@ -69,7 +70,24 @@ public class CrawlServiceImpl implements CrawlService {
crawlSourceMapper.insertSelective(source);
}
@Override
public void updateCrawlSource(CrawlSource source) {
if(source.getId()!=null){
Optional<CrawlSource> opt=crawlSourceMapper.selectByPrimaryKey(source.getId());
if(opt.isPresent()) {
CrawlSource crawlSource =opt.get();
if (crawlSource.getSourceStatus() == (byte) 1) {
//关闭
openOrCloseCrawl(crawlSource.getId(),(byte)0);
}
Date currentDate = new Date();
crawlSource.setUpdateTime(currentDate);
crawlSource.setCrawlRule(source.getCrawlRule());
crawlSource.setSourceName(source.getSourceName());
crawlSourceMapper.updateByPrimaryKey(crawlSource);
}
}
}
@Override
public PageBean<CrawlSource> listCrawlByPage(int page, int pageSize) {
PageHelper.startPage(page, pageSize);
@ -138,12 +156,17 @@ public class CrawlServiceImpl implements CrawlService {
@Override
public CrawlSource queryCrawlSource(Integer sourceId) {
SelectStatementProvider render = select(CrawlSourceDynamicSqlSupport.sourceStatus, CrawlSourceDynamicSqlSupport.crawlRule)
SelectStatementProvider render = select(id, sourceName, sourceStatus, createTime, updateTime,crawlRule)
.from(crawlSource)
.where(id, isEqualTo(sourceId))
.build()
.render(RenderingStrategies.MYBATIS3);
return crawlSourceMapper.selectMany(render).get(0);
List<CrawlSource> list= crawlSourceMapper.selectMany(render);
if(list!=null&&list.size()>0){
return list.get(0);
}
return null;
}
@Override
@ -205,6 +228,16 @@ public class CrawlServiceImpl implements CrawlService {
}
@Override
public CrawlSource getCrawlSource(Integer id) {
Optional<CrawlSource> opt=crawlSourceMapper.selectByPrimaryKey(id);
if(opt.isPresent()) {
CrawlSource crawlSource =opt.get();
return crawlSource;
}
return null;
}
/**
* 解析分类列表
*/

4
novel-crawl/src/main/resources/logback-boot.xml
View File

@ -57,8 +57,8 @@
<!-- 指定项目中某个包，当有日志操作行为时的日志记录级别 -->
<!-- com.maijinjie.springboot 为根包，也就是只要是发生在这个根包下面的所有日志操作行为的权限都是DEBUG -->
<!-- 级别依次为【从高到低】：FATAL > ERROR > WARN > INFO > DEBUG > TRACE -->
<logger name="com.java2nb" level="DEBUG">
<logger name="com.java2nb" level="DEBUG" additivity="false">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</logger>
</configuration>
</configuration>

1
novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_add.html
View File

@ -30,6 +30,7 @@
<ul class="log_list">
<li><a class="link_1" href="/">爬虫源管理</a></li>
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<li><a class="link_1" href="/crawl/crawlSource_test.html" target="_blank" >规则测试</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>

1
novel-crawl/src/main/resources/templates/crawl/crawlSingleTask_list.html
View File

@ -30,6 +30,7 @@
<ul class="log_list">
<li><a class="link_1" href="/">爬虫源管理</a></li>
<li><a class="link_1 on" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<li><a class="link_1" href="/crawl/crawlSource_test.html" target="_blank" >规则测试</a></li>
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>

1
novel-crawl/src/main/resources/templates/crawl/crawlSource_add.html
View File

@ -30,6 +30,7 @@
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<li><a class="link_1" href="/crawl/crawlSource_test.html" target="_blank" >规则测试</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>

15
novel-crawl/src/main/resources/templates/crawl/crawlSource_list.html
View File

@ -29,6 +29,7 @@
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<li><a class="link_1" href="/crawl/crawlSource_test.html" target="_blank" >规则测试</a></li>
<!-- <li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
@ -38,7 +39,7 @@
<div class="my_bookshelf">
<div class="title cf">
<h2 class="fl">爬虫源列表</h2>
<div class="fr"><a href="/crawl/crawlSource_add.html" class="btn_red">增加爬虫源</a></div>
<div class="fr"><a href="/crawl/crawlSource_add.html" class="btn_red">增加爬虫源</a>
</div>
<div id="divData" class="updateTable">
@ -119,8 +120,8 @@
<script language="javascript" type="text/javascript">
search(1, 10);
var pageCrawlSourceList=null;
function search(curr, limit) {
$.ajax({
type: "get",
url: "/crawl/listCrawlByPage",
@ -129,6 +130,7 @@
success: function (data) {
if (data.code == 200) {
var crawlSourceList = data.data.list;
pageCrawlSourceList=data.data.list;
if (crawlSourceList.length > 0) {
var crawlSourceListHtml = "";
for(var i=0;i<crawlSourceList.length;i++){
@ -147,7 +149,9 @@
" <td class=\"goread\" id='sourceStatus"+crawlSource.id+"'>"+(crawlSource.sourceStatus==0?'停止运行':'正在运行')+
" </td>\n" +
" <td class=\"goread\" id='opt"+crawlSource.id+"'><a href='javascript:openOrStopCrawl("+crawlSource.id+","+crawlSource.sourceStatus+")'>"+(crawlSource.sourceStatus==0?'开启':'关闭')+" </a></td> </tr>");
" <td class=\"goread\" id='opt"+crawlSource.id+"'><a href='javascript:openOrStopCrawl("+crawlSource.id+","+crawlSource.sourceStatus+")'>"+(crawlSource.sourceStatus==0?'开启':'关闭')+" </a>" +
"<a href='javascript:updateCrawlSource("+crawlSource.id+")'>修改 </a>" +
"</td> </tr>");
}
$("#crawlSourceList").html(crawlSourceListHtml);
@ -196,7 +200,12 @@
})
}
function updateCrawlSource(crawlSourceId){
localStorage.setItem("crawlSourceId",crawlSourceId);
window.location.href="/crawl/crawlSource_update.html";
}
function openOrStopCrawl(sourceId,status) {

171
novel-crawl/src/main/resources/templates/crawl/crawlSource_test.html Normal file
View File

@ -0,0 +1,171 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>爬虫管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
</head>
</head>
<body class="">
<div class="header">
<div class="mainNav" id="mainNav">
<div class="box_center cf"
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
小说精品屋爬虫管理
</div>
</div>
</div>
<div class="main box_center cf">
<div class="userBox cf">
<div class="my_l">
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<li><a class="link_1" href="/crawl/crawlSource_test.html" target="_blank" >规则测试</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
</div>
<div class="my_r">
<div class="my_bookshelf">
<div class="userBox cf">
<form method="post" action="./register.html" id="form2">
<div class="aspNetHidden">
<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE"
value="/wEPDwUKLTIzNjMxNDQxNw9kFgJmD2QWAmYPFgIeBFRleHQFqAE8YSBocmVmPSIvc2VhcmNoLmFzcHg/c2VhcmNoS2V5PeWWu+Wuiembr++8jOeLhOazve+8jOeBteW8gu+8jOWJjeS4luS7iueUn++8jOWGpeeOi+msvOWkqyIgdGFyZ2V0PSJfYmxhbmsiPuWWu+Wuiembr++8jOeLhOazve+8jOeBteW8gu+8jOWJjeS4luS7iueUn++8jOWGpeeOi+msvOWkqzwvYT5kZOquoASBvnvPbc/TYIQiLhSPJ8GKnYQrmk7jGhb5AC5Q">
</div>
<div class="aspNetHidden">
<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="23AA6834">
<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION"
value="/wEdAAVece19BIZ9HiByRfHz3pfnqKSXUE1UN51mNFrIuw38c3Y2+Mc6SrnAqio3oCKbxYZZ1lS+gZUZKpbsAea8j7ASAv40DHFcQ/NE7tJUnABeyQ3d9sFDIcFCYNqlVtprfLoh4JFy0U+R/CcMuyAiWTz7">
</div>
<div class="user_l">
<div></div>
<h3>爬虫源信息填写（示例均为顶点小说网：dingdiann.com）</h3>
<ul class="log_list">
<li><span id="LabErr"></span></li>
示例：<b>http://m.xdingdiann.com/sort/{catId}/{page}.html</b> ({catId}代表分类ID，{page}代表分页页码)
<li><input type="text" id="url" class="s_input icon_key"
placeholder="url"></li>
示例：<b>value=\"(\\d+)/\\d+\"</b>
<li><input type="text" id="rule" class="s_input icon_name" placeholder="规则"></li>
示例：<b>1强制刷新 空或0使用缓存</b>
<li><input type="text" id="isRefresh" class="s_input icon_name" placeholder="是否强制刷新"></li>
<li><textarea rows="20" cols="100" id="resultMap"></textarea></li>
<li><input type="button" onclick="testCrawlSource()" name="btnRegister" value="测试"
id="btnRegister" class="btn_red"></li>
</ul>
</div>
</form>
</div>
<!--<div id="divData" class="updateTable">
<table cellpadding="0" cellspacing="0">
<thead>
<tr>
<th class="name">
爬虫源（已开启的爬虫源）
</th>
<th class="chapter">
成功爬取数量（websocket实现）
</th>
<th class="time">
目标爬取数量
</th>
<th class="goread">
状态（正在运行，已停止）（一次只能运行一个爬虫源）
</th>
<th class="goread">
操作（启动，停止）
</th>
</tr>
</thead>
<tbody id="bookShelfList">
</tbody>
</table>
<div class="pageBox cf" id="shellPage">
</div>
</div>-->
</div>
</div>
</div>
</div>
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
function load() {
var testParseUrl= localStorage.getItem("testParseUrl");
$("#url").val(testParseUrl);
var testParseRule=localStorage.getItem("testParseRule");
$("#rule").val(testParseRule);
}
function testCrawlSource() {
var data = {};
var isRefresh = $("#isRefresh").val();
data.isRefresh = isRefresh;
var rule = $("#rule").val();
if (rule.length == 0) {
layer.alert("正则必填");
return false;
}
data.rule = rule;
var url = $("#url").val();
if (url.length == 0) {
layer.alert("url必填");
return false;
}
data.url = url;
localStorage.setItem("testParseUrl",url);
localStorage.setItem("testParseRule",rule);
$.ajax({
type: "POST",
url: "/crawl/testParse",
data: data,
dataType: "json",
success: function (data) {
if (data.code == 200) {
$("#resultMap").val(JSON.stringify(data.data));
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
</script>
</html>

522
novel-crawl/src/main/resources/templates/crawl/crawlSource_update.html Normal file
View File

@ -0,0 +1,522 @@
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
<meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1"/>
<title>爬虫管理系统-小说精品屋</title>
<link rel="stylesheet" href="/css/base.css?v=1"/>
<link rel="stylesheet" href="/css/user.css"/>
</head>
</head>
<body class="">
<div class="header">
<div class="mainNav" id="mainNav">
<div class="box_center cf"
style="text-align: center;height: 44px;line-height: 48px;color: #fff;font-size: 16px;">
小说精品屋爬虫管理
</div>
</div>
</div>
<div class="main box_center cf">
<div class="userBox cf">
<div class="my_l">
<ul class="log_list">
<li><a class="link_1 on" href="/">爬虫源管理</a></li>
<li><a class="link_1" href="/crawl/crawlSingleTask_list.html">单本采集管理</a></li>
<li><a class="link_1" href="/crawl/crawlSource_test.html" target="_blank" >规则测试</a></li>
<!--<li><a class="link_1 " href="/user/userinfo.html">批量小说爬取</a></li>
<li><a class="link_4 " href="/user/favorites.html">单本小说爬取</a></li>-->
</ul>
</div>
<div class="my_r">
<div class="my_bookshelf">
<div class="userBox cf">
<form method="post" action="./register.html" id="form2">
<input type="hidden" name="id" id="sourceId"/>
<div class="aspNetHidden">
<input type="hidden" name="__VIEWSTATE" id="__VIEWSTATE"
value="/wEPDwUKLTIzNjMxNDQxNw9kFgJmD2QWAmYPFgIeBFRleHQFqAE8YSBocmVmPSIvc2VhcmNoLmFzcHg/c2VhcmNoS2V5PeWWu+Wuiembr++8jOeLhOazve+8jOeBteW8gu+8jOWJjeS4luS7iueUn++8jOWGpeeOi+msvOWkqyIgdGFyZ2V0PSJfYmxhbmsiPuWWu+Wuiembr++8jOeLhOazve+8jOeBteW8gu+8jOWJjeS4luS7iueUn++8jOWGpeeOi+msvOWkqzwvYT5kZOquoASBvnvPbc/TYIQiLhSPJ8GKnYQrmk7jGhb5AC5Q">
</div>
<div class="aspNetHidden">
<input type="hidden" name="__VIEWSTATEGENERATOR" id="__VIEWSTATEGENERATOR" value="23AA6834">
<input type="hidden" name="__EVENTVALIDATION" id="__EVENTVALIDATION"
value="/wEdAAVece19BIZ9HiByRfHz3pfnqKSXUE1UN51mNFrIuw38c3Y2+Mc6SrnAqio3oCKbxYZZ1lS+gZUZKpbsAea8j7ASAv40DHFcQ/NE7tJUnABeyQ3d9sFDIcFCYNqlVtprfLoh4JFy0U+R/CcMuyAiWTz7">
</div>
<div class="user_l">
<div></div>
<h3>爬虫源信息填写（示例均为顶点小说网：dingdiann.com）</h3>
<ul class="log_list">
<li><span id="LabErr"></span></li>
示例：<b>新顶点小说网</b>
<li><input type="text" id="sourceName" class="s_input icon_name" placeholder="源站名"></li>
<!--示例：<b>https://m.xdingdiann.com/sort/0/1.html</b>
<li><input type="text" id="updateBookListUrl" class="s_input icon_key"
placeholder="小说更新列表url"></li>-->
示例：<b>http://m.xdingdiann.com/sort/{catId}/{page}.html</b> ({catId}代表分类ID，{page}代表分页页码)
<li><input type="text" id="bookListUrl" class="s_input icon_key"
placeholder="分类列表页URL规则"></li>
示例：<b>1</b>
<li><input type="text" id="catId1" class="s_input icon_key" placeholder="玄幻奇幻分类ID"></li>
示例：<b>2</b>
<li><input type="text" id="catId2" class="s_input icon_key" placeholder="武侠仙侠分类ID"></li>
示例：<b>3</b>
<li><input type="text" id="catId3" class="s_input icon_key" placeholder="都市言情分类ID"></li>
示例：<b>4</b>
<li><input type="text" id="catId4" class="s_input icon_key" placeholder="历史军事分类ID"></li>
示例：<b>5</b>
<li><input type="text" id="catId5" class="s_input icon_key" placeholder="科幻灵异分类ID"></li>
示例：<b>6</b>
<li><input type="text" id="catId6" class="s_input icon_key" placeholder="网游竞技分类ID"></li>
示例：<b>7</b>
<li><input type="text" id="catId7" class="s_input icon_key" placeholder="女生频道分类ID"></li>
示例：<b>href="/ddk(\d+)/"</b>
<li><input type="text" id="bookIdPatten" class="s_input icon_key"
placeholder="列表页小说ID正则表达式："></li>
<b>value="(\d+)/\d+"</b>
<li><input type="text" id="pagePatten" class="s_input icon_key"
placeholder="列表页当前分页页码正则表达式："></li>
<b>value="\d+/(\d+)"</b>
<li><input type="text" id="totalPagePatten" class="s_input icon_key"
placeholder="列表页分页总页数正则表达式："></li>
<b>http://m.xdingdiann.com/ddk{bookId}</b> (bookId代表小说ID)
<li><input type="text" id="bookDetailUrl" class="s_input icon_key"
placeholder="详情页URL规则："></li>
示例：<b>&lt;p class="title"&gt;([^/]+)&lt;/p&gt;</b>
<li><input type="text" id="bookNamePatten" class="s_input icon_key"
placeholder="小说名的正则表达式："></li>
示例：<b>作者：([^/]+)<</b>
<li><input type="text" id="authorNamePatten" class="s_input icon_key"
placeholder="小说作者的正则表达式："></li>
示例：<b>&lt;img src="([^>]+)"\s+onerror="this.src=</b>
<li><input type="text" id="picUrlPatten" class="s_input icon_key"
placeholder="小说图片路径的正则表达式："></li>
<b>可空，适用于图片路径为相对路径的源站，加上小说图片路径，则为完整的可访问的图片路径</b>
<li><input type="text" id="picUrlPrefix" class="s_input icon_key"
placeholder="小说图片访问路径前缀："></li>
示例：<b>状态：([^/]+)&lt;/li&gt;</b>
<li><input type="text" id="statusPatten" class="s_input icon_key"
placeholder="小说状态的正则表达式："></li>
示例：<b>连载</b>
<li><input type="text" id="bookStatus0" class="s_input icon_key"
placeholder="连载中的小说在此网站的具体表现值："></li>
示例：<b>完结</b>
<li><input type="text" id="bookStatus1" class="s_input icon_key"
placeholder="全本小说在此网站的具体表现值："></li>
示例：<b>&lt;div\s+class="score"&gt;(\d+\.\d+)分&lt;/div&gt;</b>
<li><input type="text" id="scorePatten" class="s_input icon_key"
placeholder="小说评分的正则表达式："></li>
示例：<b></b>
<li><input type="text" id="visitCountPatten" class="s_input icon_key"
placeholder="小说点击量的正则表达式："></li>
示例：<b>&lt;p class="review"&gt;</b>
<li><input type="text" id="descStart" class="s_input icon_key"
placeholder="小说简介开始截取字符串："></li>
示例：<b>&lt;/p&gt;</b>
<li><input type="text" id="descEnd" class="s_input icon_key" placeholder="小说简介结束截取字符串：">
</li>
示例：<b>更新：(\d+-\d+-\d+\s\d+:\d+:\d+)&lt;/a&gt;</b>
<li><input type="text" id="upadateTimePatten" class="s_input icon_key"
placeholder="小说更新时间的正则表达式："></li>
示例：<b>yyyy-MM-dd HH:mm:ss</b>
<li><input type="text" id="upadateTimeFormatPatten" class="s_input icon_key"
placeholder="小说更新时间在此网站的显示模式："></li>
示例：<b>http://m.xdingdiann.com/ddk{bookId}/all.html</b> (bookId代表小说ID)
<li><input type="text" id="bookIndexUrl" class="s_input icon_key"
placeholder="小说目录页的URL规则："></li>
<b>可空，适用于最新章节列表和全部章节列表在同一个页面的源站</b>
<li><input type="text" id="bookIndexStart" class="s_input icon_key"
placeholder="小说目录页内容开始截取字符串："></li>
示例：<b>&lt;a\s+style=""\s+href="/ddk\d+/(\d+)\.html"&gt;[^/]+&lt;/a&gt;</b>
<li><input type="text" id="indexIdPatten" class="s_input icon_key"
placeholder="目录页目录ID正则表达式："></li>
示例：<b>&lt;a\s+style=""\s+href="/ddk\d+/\d+\.html"&gt;([^/]+)&lt;/a&gt;</b>
<li><input type="text" id="indexNamePatten" class="s_input icon_key"
placeholder="目录页目录名的正则表达式："></li>
示例：<b>http://m.xdingdiann.com/ddk{bookId}/{indexId}.html</b>
(bookId代表小说ID,{indexId}代表目录ID)
<li><input type="text" id="bookContentUrl" class="s_input icon_key"
placeholder="小说内容页的URL规则："></li>
示例：<b>id="content"></b>
<li><input type="text" id="contentStart" class="s_input icon_key"
placeholder="小说内容开始截取字符串："></li>
示例：<b>&lt;script&gt;</b>
<li><input type="text" id="contentEnd" class="s_input icon_key"
placeholder="小说内容结束截取字符串："></li>
<li><input type="button" onclick="updateCrawlSource()" name="btnRegister" value="提交"
id="btnRegister" class="btn_red"></li>
</ul>
</div>
</form>
</div>
<!--<div id="divData" class="updateTable">
<table cellpadding="0" cellspacing="0">
<thead>
<tr>
<th class="name">
爬虫源（已开启的爬虫源）
</th>
<th class="chapter">
成功爬取数量（websocket实现）
</th>
<th class="time">
目标爬取数量
</th>
<th class="goread">
状态（正在运行，已停止）（一次只能运行一个爬虫源）
</th>
<th class="goread">
操作（启动，停止）
</th>
</tr>
</thead>
<tbody id="bookShelfList">
</tbody>
</table>
<div class="pageBox cf" id="shellPage">
</div>
</div>-->
</div>
</div>
</div>
</div>
</body>
<script src="/javascript/jquery-1.8.0.min.js" type="text/javascript"></script>
<script src="/layui/layui.all.js" type="text/javascript"></script>
<script src="/javascript/header.js" type="text/javascript"></script>
<script src="/javascript/user.js" type="text/javascript"></script>
<script language="javascript" type="text/javascript">
function load(){
var crawlSourceId = localStorage.getItem("crawlSourceId")
if(crawlSourceId!=null){
$.ajax({
type: "GET",
url: "/crawl/getCrawlSource/"+crawlSourceId,
dataType: "json",
success: function (data) {
if (data.code == 200) {
loadPage(data.data);
} else if (data.code == 1001) {
//未登录
location.href = '/user/login.html?originUrl=' + decodeURIComponent(location.href);
}else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
}
function loadPage(data){
$("#sourceId").val(data.id);
$("#sourceName").val(data.sourceName);
if(data.crawlRule){
var crawlRule= JSON.parse(data.crawlRule);
$("#bookListUrl").val(crawlRule.bookListUrl);
var catIdRule = crawlRule.catIdRule;
try{
for (var i = 1; i <= 7; i++) {
$("#catId" + i).val(catIdRule["catId" + i]);
}
}catch(e){
}
$("#bookIdPatten").val(crawlRule.bookIdPatten);
$("#pagePatten").val(crawlRule.pagePatten);
$("#totalPagePatten").val(crawlRule.totalPagePatten);
$("#bookDetailUrl").val(crawlRule.bookDetailUrl);
$("#bookNamePatten").val(crawlRule.bookNamePatten);
$("#authorNamePatten").val(crawlRule.authorNamePatten);
$("#picUrlPatten").val(crawlRule.picUrlPatten);
$("#picUrlPrefix").val(crawlRule.picUrlPrefix);
$("#statusPatten").val(crawlRule.statusPatten);
try{
var bookStatusRule = crawlRule.bookStatusRule;
var i=0;
for(var key in bookStatusRule){
$("#bookStatus" + i).val(key);
i++;
}
}catch (e) {
}
$("#scorePatten").val(crawlRule.scorePatten);
$("#visitCountPatten").val(crawlRule.visitCountPatten);
$("#descStart").val(crawlRule.descStart);
$("#descEnd").val(crawlRule.descEnd);
$("#upadateTimePatten").val(crawlRule.upadateTimePatten);
$("#upadateTimeFormatPatten").val(crawlRule.upadateTimeFormatPatten);
$("#bookIndexUrl").val(crawlRule.bookIndexUrl);
$("#bookIndexStart").val(crawlRule.bookIndexStart);
$("#indexIdPatten").val(crawlRule.indexIdPatten);
$("#indexNamePatten").val(crawlRule.indexNamePatten);
$("#bookContentUrl").val(crawlRule.bookContentUrl);
$("#contentStart").val(crawlRule.contentStart);
$("#contentEnd").val(crawlRule.contentEnd);
}
}
load();
function updateCrawlSource() {
var crawlRule = {};
var sourceId =$("#sourceId").val();
var sourceName = $("#sourceName").val();
if (sourceName.length == 0) {
layer.alert("源站名必填");
return false;
}
var bookListUrl = $("#bookListUrl").val();
if (bookListUrl.length == 0) {
layer.alert("分类列表页URL规则必填");
return false;
}
crawlRule.bookListUrl = bookListUrl;
var catIdRule = {};
for (var i = 1; i <= 7; i++) {
var catId = $("#catId" + i).val();
if (catId.length > 0) {
catIdRule["catId" + i] = catId;
}
}
if (Object.keys(catIdRule).length == 0) {
layer.alert("分类ID至少要填一项");
return false;
}
crawlRule.catIdRule = catIdRule;
var bookIdPatten = $("#bookIdPatten").val();
if (bookIdPatten.length == 0) {
layer.alert("列表页小说ID正则表达式必填");
return false;
}
crawlRule.bookIdPatten = bookIdPatten;
var pagePatten = $("#pagePatten").val();
if (pagePatten.length > 0) {
crawlRule.pagePatten = pagePatten;
}
var totalPagePatten = $("#totalPagePatten").val();
if (totalPagePatten.length > 0) {
crawlRule.totalPagePatten = totalPagePatten;
}
var bookDetailUrl = $("#bookDetailUrl").val();
if (bookDetailUrl.length == 0) {
layer.alert("详情页URL规则必填");
return false;
}
crawlRule.bookDetailUrl = bookDetailUrl;
var bookNamePatten = $("#bookNamePatten").val();
if (bookNamePatten.length == 0) {
layer.alert("小说名的正则表达式必填");
return false;
}
crawlRule.bookNamePatten = bookNamePatten;
var authorNamePatten = $("#authorNamePatten").val();
if (authorNamePatten.length == 0) {
layer.alert("小说作者的正则表达式必填");
return false;
}
crawlRule.authorNamePatten = authorNamePatten;
var picUrlPatten = $("#picUrlPatten").val();
if (picUrlPatten.length > 0) {
crawlRule.picUrlPatten = picUrlPatten;
}
var picUrlPrefix = $("#picUrlPrefix").val();
if (picUrlPrefix.length > 0) {
crawlRule.picUrlPrefix = picUrlPrefix;
}
var statusPatten = $("#statusPatten").val();
if (statusPatten.length > 0) {
crawlRule.statusPatten = statusPatten;
}
var bookStatusRule = {};
for (var i = 0; i <= 1; i++) {
var bookStatus = $("#bookStatus" + i).val();
if (bookStatus.length > 0) {
bookStatusRule[bookStatus] = i;
}
}
crawlRule.bookStatusRule = bookStatusRule;
var scorePatten = $("#scorePatten").val();
if (scorePatten.length > 0) {
crawlRule.scorePatten = scorePatten;
}
var visitCountPatten = $("#visitCountPatten").val();
if (visitCountPatten.length > 0) {
crawlRule.visitCountPatten = visitCountPatten;
}
var descStart = $("#descStart").val();
if (descStart.length == 0) {
layer.alert("小说简介开始截取字符串必填");
return false;
}
crawlRule.descStart = descStart;
var descEnd = $("#descEnd").val();
if (descEnd.length == 0) {
layer.alert("小说简介结束截取字符串必填");
return false;
}
crawlRule.descEnd = descEnd;
var upadateTimePatten = $("#upadateTimePatten").val();
if (upadateTimePatten.length > 0) {
crawlRule.upadateTimePatten = upadateTimePatten;
}
var upadateTimeFormatPatten = $("#upadateTimeFormatPatten").val();
if (upadateTimeFormatPatten.length > 0) {
crawlRule.upadateTimeFormatPatten = upadateTimeFormatPatten;
}
var bookIndexUrl = $("#bookIndexUrl").val();
if (bookIndexUrl.length == 0) {
layer.alert("小说目录页的URL规则必填");
return false;
}
crawlRule.bookIndexUrl = bookIndexUrl;
var bookIndexStart = $("#bookIndexStart").val();
if (bookIndexStart.length > 0) {
crawlRule.bookIndexStart = bookIndexStart;
}
var indexIdPatten = $("#indexIdPatten").val();
if (indexIdPatten.length == 0) {
layer.alert("小说目录页的目录ID正则表达式必填");
return false;
}
crawlRule.indexIdPatten = indexIdPatten;
var indexNamePatten = $("#indexNamePatten").val();
if (indexNamePatten.length == 0) {
layer.alert("小说目录页的目录名正则表达式必填");
return false;
}
crawlRule.indexNamePatten = indexNamePatten;
var bookContentUrl = $("#bookContentUrl").val();
if (bookContentUrl.length == 0) {
layer.alert("小说内容页的URL规则必填");
return false;
}
crawlRule.bookContentUrl = bookContentUrl;
var contentStart = $("#contentStart").val();
if (contentStart.length == 0) {
layer.alert("小说内容开始截取字符串必填");
return false;
}
crawlRule.contentStart = contentStart;
var contentEnd = $("#contentEnd").val();
if (contentEnd.length == 0) {
layer.alert("小说内容结束截取字符串必填");
return false;
}
crawlRule.contentEnd = contentEnd;
$.ajax({
type: "POST",
url: "/crawl/updateCrawlSource",
data: {'id':sourceId,'sourceName': sourceName, 'crawlRule': JSON.stringify(crawlRule)},
dataType: "json",
success: function (data) {
if (data.code == 200) {
window.location.href = '/crawl/crawlSource_list.html';
} else {
layer.alert(data.msg);
}
},
error: function () {
layer.alert('网络异常');
}
})
}
</script>
</html>

82
novel-front/pom.xml
View File

@ -87,14 +87,92 @@
</dependency>
</dependencies>
<!-- <build>-->
<!-- <plugins>-->
<!-- <plugin>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-maven-plugin</artifactId>-->
<!-- </plugin>-->
<!-- </plugins>-->
<!-- </build>-->
<build>
<plugins>
<plugin>
<!--打包时去除第三方依赖-->
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<configuration>
<layout>ZIP</layout>
<includes>
<include>
<groupId>non-exists</groupId>
<artifactId>non-exists</artifactId>
</include>
</includes>
</configuration>
</plugin>
<!--拷贝第三方依赖文件到指定目录-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
<executions>
<execution>
<id>copy-dependencies</id>
<phase>package</phase>
<goals>
<goal>copy-dependencies</goal>
</goals>
<configuration>
<!--target/lib是依赖jar包的输出目录，根据自己喜好配置-->
<outputDirectory>target/lib</outputDirectory>
<excludeTransitive>false</excludeTransitive>
<stripVersion>false</stripVersion>
<includeScope>runtime</includeScope>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId>maven-antrun-plugin</artifactId>
<version>1.8</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>run</goal>
</goals>
<configuration>
<tasks>
<!-- 文件夹 -->
<copy todir="${project.build.directory}/build/conf" overwrite="true">
<fileset dir="${basedir}/src/main/resources">
<include name="**/*.*"/>
<exclude name="mybatis/*/*.*"/>
</fileset>
</copy>
<move todir="${project.build.directory}/build/lib">
<fileset dir="target/lib"/>
</move>
<copy file="${project.build.directory}/${project.artifactId}-${project.version}.jar"
tofile="${project.build.directory}/build/${project.artifactId}.jar" />
<fixcrlf srcdir="${basedir}/src/main/build/scripts" eol="unix"/>
<copy todir="${project.build.directory}/build/bin">
<fileset dir="${basedir}/src/main/build/scripts">
<include name="*.sh" />
<include name="*.txt" />
<include name="*.bat" />
</fileset>
</copy>
<zip destfile='${project.build.directory}/build/${project.artifactId}.zip'>
<zipfileset filemode="755" dir= '${project.build.directory}/build/' />
</zip>
</tasks>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>
</project>

8
novel-front/src/main/build/scripts/readme.txt Normal file
View File

@ -0,0 +1,8 @@
1：linux启动环境
sh start.sh
3：windows启动环境
windows-start.bat
3：linux停止应用
sh stop.sh

47
novel-front/src/main/build/scripts/start.sh Normal file
View File

@ -0,0 +1,47 @@
#!/bin/bash
ENGINE=novel-front.jar
cd ../
#部署目路
DEPLOY_DIR=`pwd`
#获取到当前目录的名称
SERVER_NAME=`basename $DEPLOY_DIR`
#应用进程
PIDS=`ps -ef | grep java | grep "$ENGINE" |awk '{print $2}'`
#设置日志文件的输出目录
LOGS_DIR=$DEPLOY_DIR/logs
if [ ! -d $LOGS_DIR ]; then
mkdir $LOGS_DIR
fi
#日志
STDOUT_FILE=$LOGS_DIR/stdout.log
#JAVA 环境配置
JAVA_OPTS=" -Djava.net.preferIPv4Stack=true -Dlog.home=$LOGS_DIR"
JAVA_MEM_OPTS=" -server -Xms1024m -Xmx1024m -XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=256m -XX:+UseG1GC -XX:MaxGCPauseMillis=100 -XX:InitiatingHeapOccupancyPercent=50 -XX:+PrintGCApplicationStoppedTime -XX:+PrintGCApplicationConcurrentTime -XX:+PrintGCDateStamps -verbose:gc -XX:+PrintGCDetails -XX:+PrintHeapAtGC -Xloggc:$LOGS_DIR/gc.log -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=100M -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=dump.hprof "
#退出标志
RETVAL="0"
if [ -n "$PIDS" ]; then
echo "ERROR: The $SERVER_NAME already started!"
echo "PID: $PIDS"
exit $RETVAL
fi
nohup java -jar $JAVA_OPTS $JAVA_MEM_OPTS -Dloader.path=conf,lib $ENGINE > $STDOUT_FILE 2>&1 &
COUNT=0
while [ $COUNT -lt 1 ]; do
echo -e ".\c"
sleep 1
COUNT=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}' | wc -l`
if [ $COUNT -gt 0 ]; then
break
fi
done
echo "OK!"
PIDS=`ps -f | grep java | grep "$DEPLOY_DIR" | awk '{print $2}'`
echo "PID: $PIDS"
echo "STDOUT: $STDOUT_FILE"

33
novel-front/src/main/build/scripts/stop.sh Normal file
View File

@ -0,0 +1,33 @@
#!/bin/bash
SERVER_NAME=novel-front.jar
#应用进程
PIDS=`ps -ef | grep java | grep "$SERVER_NAME" |awk '{print $2}'`
if [ -z "$PIDS" ]; then
echo "ERROR: The $SERVER_NAME does not started!"
exit 1
fi
echo -e "Stopping the $SERVER_NAME ...\c"
for PID in $PIDS ; do
kill $PID > /dev/null 2>&1
done
COUNT=0
while [ $COUNT -lt 1 ]; do
echo -e ".\c"
sleep 1
COUNT=1
for PID in $PIDS ; do
PID_EXIST=`ps -f -p $PID | grep java`
if [ -n "$PID_EXIST" ]; then
COUNT=0
break
fi
done
done
echo "OK!"
echo "PID: $PIDS"
PIDS=""

10
novel-front/src/main/build/scripts/windows-start.bat Normal file
View File

@ -0,0 +1,10 @@
@echo off
setlocal enabledelayedexpansion
set JAVA=java
set OPTS=-XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=128m -Xms1024m -Xmx1024m -Xmn256m -Xss256k -XX:SurvivorRatio=8 -XX:+UseConcMarkSweepGC
set ENGINE=novel-front.jar
cd ../
java -jar %OPTS% -Dloader.path=conf,lib %ENGINE%
pause

4
novel-front/src/main/resources/logback-boot.xml
View File

@ -57,8 +57,8 @@
<!-- 指定项目中某个包，当有日志操作行为时的日志记录级别 -->
<!-- com.maijinjie.springboot 为根包，也就是只要是发生在这个根包下面的所有日志操作行为的权限都是DEBUG -->
<!-- 级别依次为【从高到低】：FATAL > ERROR > WARN > INFO > DEBUG > TRACE -->
<logger name="com.java2nb" level="DEBUG">
<logger name="com.java2nb" level="DEBUG" additivity="false">
<appender-ref ref="STDOUT" />
<appender-ref ref="FILE" />
</logger>
</configuration>
</configuration>

4
pom.xml
View File

@ -30,7 +30,7 @@
<java.version>1.8</java.version>
<maven.test.skip>true</maven.test.skip>
<mysql.version>8.0.11</mysql.version>
<mybatis.version>1.3.2</mybatis.version>
<mybatis.version>2.1.4</mybatis.version>
<mybatis-generator.version>1.4.0</mybatis-generator.version>
<mybatis-dynamic-sql.version>1.1.4</mybatis-dynamic-sql.version>
<pagehelper.version>1.2.5</pagehelper.version>
@ -112,4 +112,4 @@
</snapshots>
</pluginRepository>
</pluginRepositories>
</project>
</project>