更新文档

README.md

@@ -55,6 +55,8 @@ novel-plus -- 父工程
 | Redis                | 分布式缓存(缓存替换方案，默认关闭，一行配置开启)                               
 | ElasticSearch        | 搜索引擎(搜索增强方案，默认关闭，一行配置开启)                      
 | RabbitMq             | 消息队列(流量削峰，默认关闭，一行配置开启)  
+| OSS                  | 阿里云对象存储服务(图片存储方式之一，一行配置即可切换) 
+| FastDfs              |开源轻量级分布式文件系统(图片存储方式之一，一行配置即可切换)                      
 | Redisson             | 实现分布式锁                                       
 | Lombok               | 简化对象封装工具                                                                               
 | Docker               | 应用容器引擎   

novel-common/src/main/resources/application-common-dev.yml

@@ -4,7 +4,7 @@ spring:
   datasource:
     url: jdbc:mysql://127.0.0.1:3306/novel_plus?useUnicode=true&characterEncoding=utf-8&useSSL=false&serverTimezone=Asia/Shanghai
     username: root
-    password:
+    password: test123456
     driver-class-name: com.mysql.cj.jdbc.Driver
   #Redis服务器IP
   redis:
@@ -30,7 +30,8 @@ spring:
 
 pic:
   save:
-    type: 2 #图片保存方式, 1不保存，使用网络图片 ，2本地保存
+    type: 2 #图片保存方式, 1不保存，使用爬取的网络图片 ，2保存在自己的存储介质
+    storage: local #存储介质，local：本地，OSS：阿里云对象存储，fastDfs：分布式文件系统
     path: /var/pic  #图片保存路径
 
 

novel-crawl/src/main/java/com/java2nb/novel/core/listener/StarterListener.java

@@ -61,7 +61,7 @@ public class StarterListener implements ServletContextListener {
                                 Book book = CrawlParser.parseBook(ruleBean, needUpdateBook.getCrawlBookId());
                                 //这里只做老书更新
                                 book.setId(needUpdateBook.getId());
-                                if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().startsWith(Constants.LOCAL_PIC_PREFIX)) {
+                                if(needUpdateBook.getPicUrl()!=null && needUpdateBook.getPicUrl().contains(Constants.LOCAL_PIC_PREFIX)) {
                                     //本地图片则不更新
                                     book.setPicUrl(null);
                                 }

novel-front/pom.xml

@@ -39,6 +39,30 @@
             <version>${jest.version}</version>
         </dependency>
 
+        <!--aliyunOSS-->
+        <dependency>
+            <groupId>com.aliyun.oss</groupId>
+            <artifactId>aliyun-sdk-oss</artifactId>
+            <version>2.4.0</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-fileupload</groupId>
+            <artifactId>commons-fileupload</artifactId>
+            <version>1.3.1</version>
+        </dependency>
+
+        <!--FastDfs-->
+        <dependency>
+            <groupId>org.springframework</groupId>
+            <artifactId>spring-mock</artifactId>
+            <version>2.0.8</version>
+        </dependency>
+        <dependency>
+            <groupId>com.github.tobato</groupId>
+            <artifactId>fastdfs-client</artifactId>
+            <version>1.26.1-RELEASE</version>
+        </dependency>
+
         <dependency>
             <groupId>org.springframework.boot</groupId>
             <artifactId>spring-boot-starter-data-elasticsearch</artifactId>

novel-front/src/main/java/com/java2nb/novel/FrontNovelApplication.java

@@ -1,10 +1,12 @@
 package com.java2nb.novel;
 
+import com.github.tobato.fastdfs.FdfsClientConfig;
 import org.mybatis.spring.annotation.MapperScan;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.cache.annotation.EnableCaching;
 import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Import;
 import org.springframework.scheduling.TaskScheduler;
 import org.springframework.scheduling.annotation.EnableScheduling;
 import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler;
@@ -18,6 +20,7 @@ import org.springframework.transaction.annotation.EnableTransactionManagement;
 @EnableScheduling
 @EnableCaching
 @MapperScan(basePackages = {"com.java2nb.novel.mapper"})
+@Import(FdfsClientConfig.class)
 public class FrontNovelApplication {
 
     public static void main(String[] args) {

novel-front/src/main/java/com/java2nb/novel/core/config/FdfsConfiguration.java

@@ -0,0 +1,13 @@
+package com.java2nb.novel.core.config;
+
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.EnableMBeanExport;
+import org.springframework.jmx.support.RegistrationPolicy;
+
+/**
+ * @author 11797
+ */
+@Configuration
+@EnableMBeanExport(registration= RegistrationPolicy.IGNORE_EXISTING)
+public class FdfsConfiguration {
+}

novel-front/src/main/java/com/java2nb/novel/core/config/OssProperties.java

@@ -0,0 +1,30 @@
+package com.java2nb.novel.core.config;
+
+import lombok.Data;
+import org.springframework.beans.factory.InitializingBean;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * @author 11797
+ */
+@Data
+@Component
+@ConfigurationProperties(prefix="novel.file")
+public class OssProperties{
+
+    private String endpoint;
+
+    private String keyId;
+
+    private String keySecret;
+
+    private String fileHost;
+
+    private String bucketName;
+
+    private String webUrl;
+
+
+}

novel-front/src/main/java/com/java2nb/novel/core/schedule/CrawlPicTransSchedule.java

@@ -1,5 +1,6 @@
 package com.java2nb.novel.core.schedule;
 
+import com.java2nb.novel.core.utils.Constants;
 import com.java2nb.novel.entity.Book;
 import com.java2nb.novel.service.BookService;
 import lombok.RequiredArgsConstructor;
@@ -13,7 +14,7 @@ import org.springframework.stereotype.Service;
 import java.util.List;
 
 /**
- * 网络图片转存本地任务
+ * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）任务
  *
  * @author Administrator
  */
@@ -21,7 +22,7 @@ import java.util.List;
 @Service
 @RequiredArgsConstructor
 @Slf4j
-public class Network2LocalPicSchedule {
+public class CrawlPicTransSchedule {
 
     private final BookService bookService;
 
@@ -41,7 +42,7 @@ public class Network2LocalPicSchedule {
         log.info("Network2LocalPicSchedule。。。。。。。。。。。。");
 
 
-        List<Book> networkPicBooks = bookService.queryNetworkPicBooks(100);
+        List<Book> networkPicBooks = bookService.queryNetworkPicBooks(Constants.LOCAL_PIC_PREFIX,100);
         for (Book book : networkPicBooks) {
             bookService.updateBookPicToLocal(book.getPicUrl(), book.getId());
             //3秒钟转化一张图片，10分钟转化200张

novel-front/src/main/java/com/java2nb/novel/core/utils/Constants.java

@@ -12,7 +12,7 @@ public class Constants {
     public static final String TEMPLATE_PATH_PREFIX_KEY = "templatePathPrefixKey";
 
     /**
-     * 本地图片保存前缀
+     * 保存图片到自己的存储介质路径前缀
      * */
     public static final String LOCAL_PIC_PREFIX = "/localPic/";
 

novel-front/src/main/java/com/java2nb/novel/mapper/FrontBookMapper.java

@@ -22,7 +22,7 @@ public interface FrontBookMapper extends BookMapper {
 
     void addCommentCount(@Param("bookId") Long bookId);
 
-    List<Book> queryNetworkPicBooks(@Param("limit") Integer limit);
+    List<Book> queryNetworkPicBooks(@Param("localPicPrefix") String localPicPrefix, @Param("limit") Integer limit);
 
     /**
      * 按评分随机查询小说集合

novel-front/src/main/java/com/java2nb/novel/service/BookService.java

@@ -180,15 +180,17 @@ public interface BookService {
 
     /**
      * 查询网络图片的小说
+     *
+     * @param localPicPrefix
      * @param limit 查询条数
      * @return 返回小说集合
      * */
-    List<Book> queryNetworkPicBooks(Integer limit);
+    List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit);
 
 
     /**
-     * 更新小说网络图片到本地
+     * 更新爬取的小说网络图片到自己的存储介质（本地、OSS、fastDfs）
-     * @param picUrl 网络图片路径
+     * @param picUrl 爬取的网络图片路径
      * @param bookId 小说ID
      */
     void updateBookPicToLocal(String picUrl, Long bookId);
@@ -198,6 +200,7 @@ public interface BookService {
      * @param userId 用户ID
      * @param page 页码
      * @param pageSize 分页大小
+     * @return 小说集合
      * */
     List<Book> listBookPageByUserId(Long userId, int page, int pageSize);
 

novel-front/src/main/java/com/java2nb/novel/service/FileService.java

@@ -0,0 +1,17 @@
+package com.java2nb.novel.service;
+
+
+/**
+ * @author 11797
+ */
+public interface FileService {
+
+    /**
+     * 将爬取的网络图片转存为自己的存储介质（本地、OSS、fastDfs）
+     * @param picSrc 爬取的网络图片路径
+     * @param picSavePath 保存路径
+     * @return 新图片地址
+     * */
+    String transFile(String picSrc, String picSavePath);
+
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/BookServiceImpl.java

@@ -14,6 +14,7 @@ import com.java2nb.novel.mapper.*;
 import com.java2nb.novel.search.BookSP;
 import com.java2nb.novel.service.AuthorService;
 import com.java2nb.novel.service.BookService;
+import com.java2nb.novel.service.FileService;
 import com.java2nb.novel.service.SearchService;
 import com.java2nb.novel.vo.BookCommentVO;
 import com.java2nb.novel.vo.BookSettingVO;
@@ -82,6 +83,8 @@ public class BookServiceImpl implements BookService {
 
     private final SearchService searchService;
 
+    private final FileService fileService;
+
 
     @SneakyThrows
     @Override
@@ -460,14 +463,14 @@ public class BookServiceImpl implements BookService {
     }
 
     @Override
-    public List<Book> queryNetworkPicBooks(Integer limit) {
+    public List<Book> queryNetworkPicBooks(String localPicPrefix, Integer limit) {
-        return bookMapper.queryNetworkPicBooks(limit);
+        return bookMapper.queryNetworkPicBooks(localPicPrefix,limit);
     }
 
     @Override
     public void updateBookPicToLocal(String picUrl, Long bookId) {
 
-        picUrl = FileUtil.network2Local(picUrl, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        picUrl = fileService.transFile(picUrl, picSavePath);
 
         bookMapper.update(update(book)
                 .set(BookDynamicSqlSupport.picUrl)

novel-front/src/main/java/com/java2nb/novel/service/impl/FastDfsFileServiceImpl.java

@@ -0,0 +1,60 @@
+package com.java2nb.novel.service.impl;
+
+import com.github.tobato.fastdfs.domain.StorePath;
+import com.github.tobato.fastdfs.service.FastFileStorageClient;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.io.FilenameUtils;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+import java.io.FileInputStream;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@Slf4j
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "fastDfs")
+public class FastDfsFileServiceImpl implements FileService {
+
+    private final FastFileStorageClient storageClient;
+
+    @Value("${fdfs.webUrl}")
+    private String webUrl;
+
+    @Override
+    public String transFile(String picSrc, String picSavePath) {
+
+        File file;
+        String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
+            file = new File(picSavePath + filePath);
+        } else {
+            //默认图片不存储
+            return filePath;
+        }
+
+        try {
+            FileInputStream inputStream = new FileInputStream(file);
+            StorePath storePath = storageClient.uploadFile(inputStream, file.length(),
+                    FilenameUtils.getExtension(file.getName()), null);
+            //这里额外加上LOCAL_PIC_PREFIX路径，表明该图片是个人资源，而不是爬虫爬取的网络资源，不需要再次进行转换，
+            // 实际访问时，再通过nginx的rewite指令来重写路径，去掉LOCAL_PIC_PREFIX
+            return webUrl+Constants.LOCAL_PIC_PREFIX+storePath.getFullPath();
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            //删除
+            file.delete();
+        }
+
+        return "/images/default.gif";
+    }
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/LocalFileServiceImpl.java

@@ -0,0 +1,23 @@
+package com.java2nb.novel.service.impl;
+
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "local")
+public class LocalFileServiceImpl implements FileService {
+
+    @Override
+    public String transFile(String picSrc, String picSavePath){
+
+        return FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+    }
+}

novel-front/src/main/java/com/java2nb/novel/service/impl/OssFileServiceImpl.java

@@ -0,0 +1,76 @@
+package com.java2nb.novel.service.impl;
+
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.model.CannedAccessControlList;
+import com.aliyun.oss.model.CreateBucketRequest;
+import com.aliyun.oss.model.PutObjectRequest;
+import com.aliyun.oss.model.PutObjectResult;
+import com.java2nb.novel.core.config.OssProperties;
+import com.java2nb.novel.core.utils.Constants;
+import com.java2nb.novel.core.utils.FileUtil;
+import com.java2nb.novel.service.FileService;
+import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.io.File;
+
+/**
+ * @author 11797
+ */
+@Service
+@RequiredArgsConstructor
+@ConditionalOnProperty(prefix = "pic.save", name = "storage", havingValue = "OSS")
+@Slf4j
+public class OssFileServiceImpl implements FileService {
+
+    private final OssProperties ossProperties;
+
+    @Override
+    public String transFile(String picSrc, String picSavePath) {
+
+        File file;
+        String filePath = FileUtil.network2Local(picSrc, picSavePath, Constants.LOCAL_PIC_PREFIX);
+        if (filePath.contains(Constants.LOCAL_PIC_PREFIX)) {
+            file = new File(picSavePath+filePath);
+        } else {
+            //默认图片不存储
+            return filePath;
+        }
+
+        filePath = filePath.replaceFirst(picSavePath,"");
+
+        filePath = filePath.startsWith("/") ? filePath.replaceFirst("/","") : filePath;
+
+
+        OSSClient ossClient = new OSSClient(ossProperties.getEndpoint(), ossProperties.getKeyId(), ossProperties.getKeySecret());
+        try {
+            //容器不存在，就创建
+            if (!ossClient.doesBucketExist(ossProperties.getBucketName())) {
+                ossClient.createBucket(ossProperties.getBucketName());
+                CreateBucketRequest createBucketRequest = new CreateBucketRequest(ossProperties.getBucketName());
+                createBucketRequest.setCannedACL(CannedAccessControlList.PublicRead);
+                ossClient.createBucket(createBucketRequest);
+            }
+            //上传文件
+            PutObjectResult result = ossClient.putObject(new PutObjectRequest(ossProperties.getBucketName(), filePath, file));
+            //设置权限 这里是公开读
+            ossClient.setBucketAcl(ossProperties.getBucketName(), CannedAccessControlList.PublicRead);
+
+            if(result != null) {
+                return ossProperties.getWebUrl() + "/" + filePath;
+            }
+        } catch (Exception e) {
+            log.error(e.getMessage(), e);
+        } finally {
+            //关闭
+            ossClient.shutdown();
+            file.delete();
+        }
+
+        return "/images/default.gif";
+    }
+
+
+}

novel-front/src/main/resources/application-fastdfs.yml

@@ -0,0 +1,16 @@
+#FastDfs的配置	====================================
+#读取inputsream阻塞时间
+fdfs:
+  connect-timeout: 600
+  so-timeout: 1500
+  #tracker地址
+  trackerList: 127.0.0.1:22122
+  #缩略图配置
+  thumbImage:
+    height: 150
+    width: 150
+  #通过nginx 访问地址
+  webUrl: http://127.0.0.1/
+  #获取连接池最大数量
+  pool.max-total: 200
+

novel-front/src/main/resources/application-oss.yml

@@ -0,0 +1,9 @@
+#OSS
+novel:
+  file:
+    endpoint: oss-cn-shenzhen.aliyuncs.com #不同的服务器，地址不同
+    key-id: dhjjrue6767778878 #去OSS控制台获取
+    key-secret: dssdkkrkelrkellk44554 #去OSS控制台获取
+    bucket-name: novel #这个自己创建bucket时的命名，控制台创建也行，代码创建也行
+    file-host: pic #文件路径
+    web-url: https://xxyimg.oss-cn-hangzhou.aliyuncs.com #OSS文件的web访问地址

novel-front/src/main/resources/application.yml

@@ -4,7 +4,7 @@ server:
 spring:
   profiles:
     active: dev
-    include: alipay
+    include: alipay,oss,fastdfs
 
 
   rabbitmq:

novel-front/src/main/resources/mybatis/mapping/BookMapper.xml

@@ -56,6 +56,7 @@
         select
         id,pic_url from book
         where pic_url like 'http%'
+        and pic_url not like concat('%',#{localPicPrefix},'%')
         limit #{limit}
     </select>