diff --git a/novel-common/src/main/resources/application-common-dev.yml b/novel-common/src/main/resources/application-common-dev.yml index d873e71..e5c7cdf 100644 --- a/novel-common/src/main/resources/application-common-dev.yml +++ b/novel-common/src/main/resources/application-common-dev.yml @@ -4,7 +4,7 @@ spring: datasource: url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai username: root - password: + password: test123456 driver-class-name: com.mysql.cj.jdbc.Driver #Redis服务器IP redis: @@ -30,7 +30,8 @@ spring: pic: save: - type: 2 #图片保存方式, 1不保存，使用网络图片 ，2本地保存 + type: 2 #图片保存方式, 1不保存，使用爬取的网络图片 ，2保存在自己的存储介质 + storage: local #存储介质，local：本地，OSS：阿里云对象存储，fastDfs：分布式文件系统 path: /var/pic #图片保存路径 diff --git a/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java b/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java index 6a81f3b..0e26ad5 100644 --- a/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java +++ b/novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java @@ -61,7 +61,7 @@ public class StarterListener implements ServletContextListener { Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId()); //这里只做老书更新 book.setId(needUpdateBook.getId()); - if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) { + if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) { //本地图片则不更新 book.setPicUrl(null); } diff --git a/novel-front/pom.xml b/novel-front/pom.xml index 862eef9..74fbe24 100644 --- a/novel-front/pom.xml +++ b/novel-front/pom.xml @@ -39,6 +39,18 @@ ${jest.version} + + + com.aliyun.oss + aliyun-sdk-oss + 2.4.0 + + + commons-fileupload + commons-fileupload + 1.3.1 + + org.springframework.boot spring-boot-starter-data-elasticsearch diff --git a/novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java b/novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java new file mode 100644 index 0000000..0acf3a3 --- /dev/null +++ b/novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java @@ -0,0 +1,30 @@ +package com.java2nb.novel.core.config; + +import lombok.Data; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +/** + * @author 11797 + */ +@Data +@Component +@ConfigurationProperties(prefix="novel.file") +public class OssProperties{ + + private String endpoint; + + private String keyId; + + private String keySecret; + + private String fileHost; + + private String bucketName; + + private String webUrl; + + +} diff --git a/novel-front/src/main/java/com/java2nb/novel/core/schedule/Network2LocalPicSchedule.java b/novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java similarity index 86% rename from novel-front/src/main/java/com/java2nb/novel/core/schedule/Network2LocalPicSchedule.java rename to novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java index 17a9e87..0ecae59 100644 --- a/novel-front/src/main/java/com/java2nb/novel/core/schedule/Network2LocalPicSchedule.java +++ b/novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java @@ -1,5 +1,6 @@ package com.java2nb.novel.core.schedule; +import com.java2nb.novel.core.utils.Constants; import com.java2nb.novel.entity.Book; import com.java2nb.novel.service.BookService; import lombok.RequiredArgsConstructor; @@ -13,7 +14,7 @@ import org.springframework.stereotype.Service; import java.util.List; /** - * 网络图片转存本地任务 + * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）任务 * * @author Administrator */ @@ -21,7 +22,7 @@ import java.util.List; @Service @RequiredArgsConstructor @Slf4j -public class Network2LocalPicSchedule { +public class CrawlPicTransSchedule { private final BookService bookService; @@ -41,7 +42,7 @@ public class Network2LocalPicSchedule { log.info("Network2LocalPicSchedule。。。。。。。。。。。。"); - List networkPicBooks = bookService.queryNetworkPicBooks(100); + List networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100); for (Book book : networkPicBooks) { bookService.updateBookPicToLocal(book.getPicUrl(), book.getId()); //3秒钟转化一张图片，10分钟转化200张 diff --git a/novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java b/novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java index d00ae4b..7736233 100644 --- a/novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java +++ b/novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java @@ -12,7 +12,7 @@ public class Constants { public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey"; /** - * 本地图片保存前缀 + * 保存图片到自己的存储介质路径前缀 * */ public static final String LOCAL_PIC_PREFIX = "/localPic/"; diff --git a/novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java b/novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java index 3b284ec..f71ee81 100644 --- a/novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java +++ b/novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java @@ -22,7 +22,7 @@ public interface FrontBookMapper extends BookMapper { void addCommentCount(@Param("bookId") Long bookId); - List queryNetworkPicBooks(@Param("limit") Integer limit); + List queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit); /** * 按评分随机查询小说集合 diff --git a/novel-front/src/main/java/com/java2nb/novel/service/BookService.java b/novel-front/src/main/java/com/java2nb/novel/service/BookService.java index 3e7889b..18e67ff 100644 --- a/novel-front/src/main/java/com/java2nb/novel/service/BookService.java +++ b/novel-front/src/main/java/com/java2nb/novel/service/BookService.java @@ -180,15 +180,17 @@ public interface BookService { /** * 查询网络图片的小说 + * + * @param localPicPrefix * @param limit 查询条数 * @return 返回小说集合 * */ - List queryNetworkPicBooks(Integer limit); + List queryNetworkPicBooks(String localPicPrefix, Integer limit); /** - * 更新小说网络图片到本地 - * @param picUrl 网络图片路径 + * 更新爬取的小说网络图片到自己的存储介质（本地、OSS、fastDfs） + * @param picUrl 爬取的网络图片路径 * @param bookId 小说ID */ void updateBookPicToLocal(String picUrl, Long bookId); @@ -198,6 +200,7 @@ public interface BookService { * @param userId 用户ID * @param page 页码 * @param pageSize 分页大小 + * @return 小说集合 * */ List listBookPageByUserId(Long userId, int page, int pageSize); diff --git a/novel-front/src/main/java/com/java2nb/novel/service/FileService.java b/novel-front/src/main/java/com/java2nb/novel/service/FileService.java new file mode 100644 index 0000000..d511f00 --- /dev/null +++ b/novel-front/src/main/java/com/java2nb/novel/service/FileService.java @@ -0,0 +1,17 @@ +package com.java2nb.novel.service; + + +/** + * @author 11797 + */ +public interface FileService { + + /** + * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs） + * @param picSrc 爬取的网络图片路径 + * @param picSavePath 保存路径 + * @return 新图片地址 + * */ + String transFile(String picSrc, String picSavePath); + +} diff --git a/novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java b/novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java index 3ffe180..78d6434 100644 --- a/novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java +++ b/novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java @@ -14,6 +14,7 @@ import com.java2nb.novel.mapper.*; import com.java2nb.novel.search.BookSP; import com.java2nb.novel.service.AuthorService; import com.java2nb.novel.service.BookService; +import com.java2nb.novel.service.FileService; import com.java2nb.novel.service.SearchService; import com.java2nb.novel.vo.BookCommentVO; import com.java2nb.novel.vo.BookSettingVO; @@ -82,6 +83,8 @@ public class BookServiceImpl implements BookService { private final SearchService searchService; + private final FileService fileService; + @SneakyThrows @Override @@ -460,14 +463,14 @@ public class BookServiceImpl implements BookService { } @Override - public List queryNetworkPicBooks(Integer limit) { - return bookMapper.queryNetworkPicBooks(limit); + public List queryNetworkPicBooks(String localPicPrefix, Integer limit) { + return bookMapper.queryNetworkPicBooks(localPicPrefix,limit); } @Override public void updateBookPicToLocal(String picUrl, Long bookId) { - picUrl = FileUtil.network2Local(picUrl, picSavePath, Constants.LOCAL_PIC_PREFIX); + picUrl = fileService.transFile(picUrl, picSavePath); bookMapper.update(update(book) .set(BookDynamicSqlSupport.picUrl) diff --git a/novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java b/novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java new file mode 100644 index 0000000..b356609 --- /dev/null +++ b/novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java @@ -0,0 +1,20 @@ +package com.java2nb.novel.service.impl; + +import com.java2nb.novel.service.FileService; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +/** + * @author 11797 + */ +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs") +public class FastDfsFileServiceImpl implements FileService { + + @Override + public String transFile(String picSrc, String picSavePath) { + return null; + } +} diff --git a/novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java b/novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java new file mode 100644 index 0000000..ec54c9c --- /dev/null +++ b/novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java @@ -0,0 +1,23 @@ +package com.java2nb.novel.service.impl; + +import com.java2nb.novel.core.utils.Constants; +import com.java2nb.novel.core.utils.FileUtil; +import com.java2nb.novel.service.FileService; +import lombok.RequiredArgsConstructor; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +/** + * @author 11797 + */ +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local") +public class LocalFileServiceImpl implements FileService { + + @Override + public String transFile(String picSrc, String picSavePath){ + + return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX); + } +} diff --git a/novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java b/novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java new file mode 100644 index 0000000..33d9839 --- /dev/null +++ b/novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java @@ -0,0 +1,76 @@ +package com.java2nb.novel.service.impl; + +import com.aliyun.oss.OSSClient; +import com.aliyun.oss.model.CannedAccessControlList; +import com.aliyun.oss.model.CreateBucketRequest; +import com.aliyun.oss.model.PutObjectRequest; +import com.aliyun.oss.model.PutObjectResult; +import com.java2nb.novel.core.config.OssProperties; +import com.java2nb.novel.core.utils.Constants; +import com.java2nb.novel.core.utils.FileUtil; +import com.java2nb.novel.service.FileService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.io.File; + +/** + * @author 11797 + */ +@Service +@RequiredArgsConstructor +@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS") +@Slf4j +public class OssFileServiceImpl implements FileService { + + private final OssProperties ossProperties; + + @Override + public String transFile(String picSrc, String picSavePath) { + + File file; + String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX); + if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) { + file = new File(picSavePath+filePath); + } else { + //默认图片不存储 + return filePath; + } + + filePath = filePath.replaceFirst(picSavePath,""); + + filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath; + + + OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret()); + try { + //容器不存在，就创建 + if (!ossClient.doesBucketExist(ossProperties.getBucketName())) { + ossClient.createBucket(ossProperties.getBucketName()); + CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName()); + createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead); + ossClient.createBucket(createBucketRequest); + } + //上传文件 + PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file)); + //设置权限 这里是公开读 + ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead); + + if(result != null) { + return ossProperties.getWebUrl() + "/" + filePath; + } + } catch (Exception e) { + log.error(e.getMessage(), e); + } finally { + //关闭 + ossClient.shutdown(); + file.delete(); + } + + return "/images/default.gif"; + } + + +} diff --git a/novel-front/src/main/resources/application-oss.yml b/novel-front/src/main/resources/application-oss.yml new file mode 100644 index 0000000..e0ef760 --- /dev/null +++ b/novel-front/src/main/resources/application-oss.yml @@ -0,0 +1,9 @@ +#OSS +novel: + file: + endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器，地址不同 + key-id: dhjjrue6767778878 #去OSS控制台获取 + key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取 + bucket-name: novel #这个自己创建bucket时的命名，控制台创建也行，代码创建也行 + file-host: pic #文件路径 + web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址 \ No newline at end of file diff --git a/novel-front/src/main/resources/application.yml b/novel-front/src/main/resources/application.yml index bbe93f2..a75d6bb 100644 --- a/novel-front/src/main/resources/application.yml +++ b/novel-front/src/main/resources/application.yml @@ -4,7 +4,7 @@ server: spring: profiles: active: dev - include: alipay + include: alipay,oss rabbitmq: diff --git a/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml b/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml index 5b76041..dfc1171 100644 --- a/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml +++ b/novel-front/src/main/resources/mybatis/mapping/BookMapper.xml @@ -56,6 +56,7 @@ select id,pic_url from book where pic_url like 'http%' + and pic_url not like concat('%',#{localPicPrefix},'%') limit #{limit}