201206030/novel-plus
1
0
Fork 0
mirror of https://github.com/201206030/novel-plus.git synced 2025-04-27 01:30:51 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

集成OSS对象存储

Browse Source
This commit is contained in:
xiongxiaoyang 2020-05-26 11:31:30 +08:00
parent 9975faed01
commit c62acc288e
16 changed files with 211 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
novel-common/src/main/resources/application-common-dev.yml
View File

@ -4,7 +4,7 @@ spring:
datasource:
url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
username: root
password:
password: test123456
driver-class-name: com.mysql.cj.jdbc.Driver
#Redis服务器IP
redis:
@ -30,7 +30,8 @@ spring:
pic:
save:
type: 2 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
type: 2 #图片保存方式, 1不保存，使用爬取的网络图片 ，2保存在自己的存储介质
storage: local #存储介质，local：本地，OSS：阿里云对象存储，fastDfs：分布式文件系统
path: /var/pic #图片保存路径

2
novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java
View File

@ -61,7 +61,7 @@ public class StarterListener implements ServletContextListener {
Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
//这里只做老书更新
book.setId(needUpdateBook.getId());
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
//本地图片则不更新
book.setPicUrl(null);
}

12
novel-front/pom.xml
View File

@ -39,6 +39,18 @@
<version>${jest.version}</version>
</dependency>
<!--aliyunOSS-->
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>

30
novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java Normal file
View File

@ -0,0 +1,30 @@
package com.java2nb.novel.core.config;
import lombok.Data;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
/**
* @author 11797
*/
@Data
@Component
@ConfigurationProperties(prefix="novel.file")
public class OssProperties{
private String endpoint;
private String keyId;
private String keySecret;
private String fileHost;
private String bucketName;
private String webUrl;
}

7
novel-front/src/main/java/com/java2nb/novel/core/schedule/Network2LocalPicSchedule.java → novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java
View File

@ -1,5 +1,6 @@
package com.java2nb.novel.core.schedule;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.entity.Book;
import com.java2nb.novel.service.BookService;
import lombok.RequiredArgsConstructor;
@ -13,7 +14,7 @@ import org.springframework.stereotype.Service;
import java.util.List;
/**
* 网络图片转存本地任务
* 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）任务
*
* @author Administrator
*/
@ -21,7 +22,7 @@ import java.util.List;
@Service
@RequiredArgsConstructor
@Slf4j
public class Network2LocalPicSchedule {
public class CrawlPicTransSchedule {
private final BookService bookService;
@ -41,7 +42,7 @@ public class Network2LocalPicSchedule {
log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
List<Book> networkPicBooks = bookService.queryNetworkPicBooks(100);
List<Book> networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100);
for (Book book : networkPicBooks) {
bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
//3秒钟转化一张图片，10分钟转化200张

2
novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java
View File

@ -12,7 +12,7 @@ public class Constants {
public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey";
/**
* 本地图片保存前缀
* 保存图片到自己的存储介质路径前缀
* */
public static final String LOCAL_PIC_PREFIX = "/localPic/";

2
novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java
View File

@ -22,7 +22,7 @@ public interface FrontBookMapper extends BookMapper {
void addCommentCount(@Param("bookId") Long bookId);
List<Book> queryNetworkPicBooks(@Param("limit") Integer limit);
List<Book> queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit);
/**
* 按评分随机查询小说集合

9
novel-front/src/main/java/com/java2nb/novel/service/BookService.java
View File

@ -180,15 +180,17 @@ public interface BookService {
/**
* 查询网络图片的小说
*
* @param localPicPrefix
* @param limit 查询条数
* @return 返回小说集合
* */
List<Book> queryNetworkPicBooks(Integer limit);
List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit);
/**
* 更新小说网络图片到本地
* @param picUrl 网络图片路径
* 更新爬取的小说网络图片到自己的存储介质（本地、OSS、fastDfs）
* @param picUrl 爬取的网络图片路径
* @param bookId 小说ID
*/
void updateBookPicToLocal(String picUrl, Long bookId);
@ -198,6 +200,7 @@ public interface BookService {
* @param userId 用户ID
* @param page 页码
* @param pageSize 分页大小
* @return 小说集合
* */
List<Book> listBookPageByUserId(Long userId, int page, int pageSize);

17
novel-front/src/main/java/com/java2nb/novel/service/FileService.java Normal file
View File

@ -0,0 +1,17 @@
package com.java2nb.novel.service;
/**
* @author 11797
*/
public interface FileService {
/**
* 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）
* @param picSrc 爬取的网络图片路径
* @param picSavePath 保存路径
* @return 新图片地址
* */
String transFile(String picSrc, String picSavePath);
}

9
novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java
View File

@ -14,6 +14,7 @@ import com.java2nb.novel.mapper.*;
import com.java2nb.novel.search.BookSP;
import com.java2nb.novel.service.AuthorService;
import com.java2nb.novel.service.BookService;
import com.java2nb.novel.service.FileService;
import com.java2nb.novel.service.SearchService;
import com.java2nb.novel.vo.BookCommentVO;
import com.java2nb.novel.vo.BookSettingVO;
@ -82,6 +83,8 @@ public class BookServiceImpl implements BookService {
private final SearchService searchService;
private final FileService fileService;
@SneakyThrows
@Override
@ -460,14 +463,14 @@ public class BookServiceImpl implements BookService {
}
@Override
public List<Book> queryNetworkPicBooks(Integer limit) {
return bookMapper.queryNetworkPicBooks(limit);
public List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit) {
return bookMapper.queryNetworkPicBooks(localPicPrefix,limit);
}
@Override
public void updateBookPicToLocal(String picUrl, Long bookId) {
picUrl = FileUtil.network2Local(picUrl, picSavePath, Constants.LOCAL_PIC_PREFIX);
picUrl = fileService.transFile(picUrl, picSavePath);
bookMapper.update(update(book)
.set(BookDynamicSqlSupport.picUrl)

20
novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java Normal file
View File

@ -0,0 +1,20 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.service.FileService;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs")
public class FastDfsFileServiceImpl implements FileService {
@Override
public String transFile(String picSrc, String picSavePath) {
return null;
}
}

23
novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java Normal file
View File

@ -0,0 +1,23 @@
package com.java2nb.novel.service.impl;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.service.FileService;
import lombok.RequiredArgsConstructor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local")
public class LocalFileServiceImpl implements FileService {
@Override
public String transFile(String picSrc, String picSavePath){
return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
}
}

76
novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java Normal file
View File

@ -0,0 +1,76 @@
package com.java2nb.novel.service.impl;
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.CannedAccessControlList;
import com.aliyun.oss.model.CreateBucketRequest;
import com.aliyun.oss.model.PutObjectRequest;
import com.aliyun.oss.model.PutObjectResult;
import com.java2nb.novel.core.config.OssProperties;
import com.java2nb.novel.core.utils.Constants;
import com.java2nb.novel.core.utils.FileUtil;
import com.java2nb.novel.service.FileService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.io.File;
/**
* @author 11797
*/
@Service
@RequiredArgsConstructor
@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS")
@Slf4j
public class OssFileServiceImpl implements FileService {
private final OssProperties ossProperties;
@Override
public String transFile(String picSrc, String picSavePath) {
File file;
String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
file = new File(picSavePath+filePath);
} else {
//默认图片不存储
return filePath;
}
filePath = filePath.replaceFirst(picSavePath,"");
filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath;
OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret());
try {
//容器不存在，就创建
if (!ossClient.doesBucketExist(ossProperties.getBucketName())) {
ossClient.createBucket(ossProperties.getBucketName());
CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName());
createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
ossClient.createBucket(createBucketRequest);
}
//上传文件
PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file));
//设置权限 这里是公开读
ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead);
if(result != null) {
return ossProperties.getWebUrl() + "/" + filePath;
}
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
//关闭
ossClient.shutdown();
file.delete();
}
return "/images/default.gif";
}
}

9
novel-front/src/main/resources/application-oss.yml Normal file
View File

@ -0,0 +1,9 @@
#OSS
novel:
file:
endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器，地址不同
key-id: dhjjrue6767778878 #去OSS控制台获取
key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取
bucket-name: novel #这个自己创建bucket时的命名，控制台创建也行，代码创建也行
file-host: pic #文件路径
web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址

2
novel-front/src/main/resources/application.yml
View File

@ -4,7 +4,7 @@ server:
spring:
profiles:
active: dev
include: alipay
include: alipay,oss
rabbitmq:

1
novel-front/src/main/resources/mybatis/mapping/BookMapper.xml
View File

@ -56,6 +56,7 @@
select
id,pic_url from book
where pic_url like 'http%'
and pic_url not like concat('%',#{localPicPrefix},'%')
limit #{limit}
</select>