补充注释;完善文件上传及文件清理

ReadMe.md

@@ -14,3 +14,36 @@
 ## `npm unpublish <包名(@版本)>` 删除指定包的依赖(指定版本)
 
 ## `npm i free-midway-component@latest` 项目中升级最新版本
+
+## 使用说明:
+### 引入工具
+#### upload: 文件上传功能
+##### 自行修改的属性:
+|属性名|说明|默认值|可选值|
+|:-:|:-:|:-:|:-:|
+|use|是否使用|true|false|
+|modelNames|涉及文件上传的表|-|[写表名]|
+|fileSize|上传允许文件的最大值|100mb|随便写|
+|whitelist|后缀白名单|['.jpg','.jpeg','.png','.gif','.bmp','.wbmp','.webp','.tif','.tiff','.psd','.svg','.xml','.pdf','.zip','.gz','.gzip','.mp3','.mp4','.avi']|[随便写]|
+|tempdir|临时上传路径|`join(process.cwd(), 'uploadTemp')`|是存储位置就行|
+|realdir|实际上传存储地址|`join(process.cwd(), 'upload')`|是存储位置就行|
+
+##### 使用
+  上传: ${项目前缀 koa.globalPrefix}/api/files/xxxxx/upload
+  读取/下载: ${项目前缀 koa.globalPrefix}/files/xxxx
+
+#### bull: 任务队列
+任务队列强依赖于redis,redis版本>=5;
+config中添加:
+```
+bull: {
+    defaultQueueOptions: {
+      redis: {
+        port: 6379,
+        host: '127.0.0.1',
+        // password: 'xxxxxx'
+      },
+    },
+  },
+```
+每天执行一次

package.json

@@ -40,6 +40,7 @@
   },
   "dependencies": {
     "@midwayjs/axios": "^3.9.0",
+    "@midwayjs/bull": "^3.12.3",
     "@midwayjs/upload": "^3.12.7",
     "@types/mime": "^3.0.2",
     "mime": "^3.0.0"

src/config/config.default.ts

@@ -1,5 +1,11 @@
 import { uploadWhiteList } from '@midwayjs/upload';
 import { join } from 'path';
+/**
+ * upload: {
+ *    use: 是否使用,默认为true
+ *    modelNames: 涉及文件上传的表,['${表名}',...] 会扫描这些表,然后做清理
+ * }
+ */
 export default {
   upload: {
     // mode: UploadMode, 默认为file，即上传到服务器临时目录，可以配置为 stream
@@ -10,9 +16,14 @@ export default {
     whitelist: uploadWhiteList,
     // tmpdir: string，上传的文件临时存储路径
     tmpdir: join(process.cwd(), 'uploadTemp'),
+    // realdir: string 真实上传路径
+    realdir: join(process.cwd(), 'upload'),
     // cleanTimeout: number，上传的文件在临时目录中多久之后自动删除，默认为 5 分钟
     cleanTimeout: 5 * 60 * 1000,
     // base64: boolean，设置原始body是否是base64格式，默认为false，一般用于腾讯云的兼容
     base64: false,
+    // 自定义属性
+    use: true, //默认使用
+    // modelNames: ['User'], 每个项目需要设置的.字符串数组,写表名,处理的时候直接就将名字换成model,用来操作数据库.
   },
 };

src/configuration.ts

@@ -8,8 +8,11 @@ import { ResponseMiddleware } from './middleware/response.middleware';
 import { DefaultErrorFilter } from './filter/default.filter';
 import * as axios from '@midwayjs/axios';
 import { ServiceError, FrameworkErrorEnum } from './error/service.error';
-import { IMidwayContainer } from '@midwayjs/core';
+import { IMidwayContainer, Inject, MidwayConfigService } from '@midwayjs/core';
 import * as upload from '@midwayjs/upload';
+import { UseFile } from './entity/useFile.entity';
+import * as bull from '@midwayjs/bull';
+
 
 const axiosResponse = response => {
   if (response.status === 200) return response.data;
@@ -20,9 +23,18 @@ const axiosResponse = response => {
 const axiosError = error => {
   return Promise.reject(error);
 };
+/**
+ * 框架设置:
+ *  引入
+ *      koa:基础框架,
+ *      typegoose:数据库相关,
+ *      axios:自定义http请求工具,
+ *      upload:文件上传,
+ *      bull:任务队列
+ */
 @Configuration({
   namespace: 'free',
-  imports: [koa, typegoose, axios, upload],
+  imports: [koa, typegoose, axios, upload, bull],
   importConfigs: [
     {
       default: DefaultConfig,
@@ -32,6 +44,26 @@ const axiosError = error => {
 export class FreeConfiguration {
   @App()
   app: koa.Application;
+
+  @Inject()
+  configService: MidwayConfigService;
+  /**修改配置,如果启用文件服务,那么就把项目文件地址表加上 */
+  async onConfigLoad() {
+    // 添加useFile表,将default连接中所有的文件都统计在该表中
+    const allConfig = this.configService.getConfiguration();
+    const useFile = allConfig.upload.use || false;
+    if (!useFile) return;
+    const mongoose = allConfig.mongoose;
+    if (!mongoose) return;
+    const dataSource = mongoose.dataSource;
+    if (!dataSource) return;
+    const defaultDataSource = dataSource.default;
+    if (!defaultDataSource) return;
+    let entities = defaultDataSource.entities;
+    entities = [...entities, UseFile];
+    return { mongoose: { dataSource: { default: { entities } } } };
+  }
+
   getApiUrl() {
     const path = this.app.getConfig()?.swagger?.swaggerPath;
     const port = this.app.getConfig()?.koa?.port;

src/controller/BaseController.ts

@@ -8,13 +8,44 @@ export abstract class BaseController {
   app: Application;
   @Inject()
   ctx: Context;
-
+  /**
+   * 创建
+   * @param args
+   */
   abstract create(...args);
+  /**
+   * 查询
+   * @param args
+   */
   abstract query(...args);
+  /**
+   * 单查询
+   * @param args
+   */
   abstract fetch(...args);
+  /**
+   * 修改
+   * @param args
+   */
   abstract update(...args);
+  /**
+   * 删除
+   * @param args
+   */
   abstract delete(...args);
+  /**
+   * 多修改
+   * @param args
+   */
   abstract updateMany(...args);
+  /**
+   * 多删除
+   * @param args
+   */
   abstract deleteMany(...args);
+  /**
+   * 多创建
+   * @param args
+   */
   abstract createMany(...args);
 }

src/controller/file.controller.ts

@@ -1,9 +1,11 @@
 import { Inject, Controller, Post, Files, Fields, Config, Get } from '@midwayjs/decorator';
 import { Context } from '@midwayjs/koa';
 import { join, sep } from 'path';
-import { FileService } from '../service/file.service';
+import { FileService } from '../service/File.service';
 import { createReadStream } from 'fs';
 import { getType } from 'mime';
+
+/**文件上传默认类 */
 @Controller('/')
 export class FileController {
   @Inject()
@@ -12,8 +14,8 @@ export class FileController {
   fileService: FileService;
   @Config('koa.globalPrefix')
   globalPrefix: string;
-
-  uploadDir = 'upload';
+  @Config('upload.realdir')
+  uploadDir;
 
   /**
    * 文件上传
@@ -33,8 +35,8 @@ export class FileController {
       const subs = catalog.split('_');
       dirs.push(...subs);
     }
-    let path = join(process.cwd(), this.uploadDir);
-    // TODO: 检查分级目录是否存在，不存在则创建
+    let path = this.uploadDir;
+    // 检查分级目录是否存在，不存在则创建
     for (let i = 0; i < dirs.length; i++) {
       const p = `${path}${sep}${dirs.slice(0, i + 1).join(sep)}`;
       this.fileService.mkdir(p);
@@ -48,6 +50,10 @@ export class FileController {
     this.fileService.moveFile(file.data, `${path}${filename}`);
     return { id: filename, name: filename, uri };
   }
+
+  /**
+   * 读取文件
+   */
   @Get('/files/*')
   async readFile() {
     const shortRealPath = this.fileService.getFileShortRealPath();

src/entity/BaseModel.ts

@@ -1,5 +1,9 @@
 import { modelOptions, plugin, Severity } from '@typegoose/typegoose';
 import meta from './meta';
+
+/**
+ * 表的基类,有meta字段设置及允许混合设置
+ */
 @modelOptions({
   options: { allowMixed: Severity.ALLOW },
 })

src/entity/fileType.ts

@@ -0,0 +1,31 @@
+/**自定义数据类型: 文件类型 */
+import mongoose from 'mongoose';
+interface file {
+  id: string;
+  name: string;
+  uri: string;
+}
+function instanceOfFile(object: object): object is file {
+  const have_id = 'id' in object;
+  const have_name = 'name' in object;
+  const have_uri = 'uri' in object;
+  return have_id && have_name && have_uri;
+}
+/**数据库类型-File类型
+ * e.g.:[{id:string,name:string,uri:string}]
+ */
+class File extends mongoose.SchemaType {
+  constructor(key, options) {
+    super(key, options, 'File');
+  }
+  cast(val: Array<object>) {
+    for (const o of val) {
+      const result = instanceOfFile(o);
+      if (!result) throw new mongoose.Error.ObjectParameterError('file object properties are miss, should have: id name uri');
+    }
+    return val;
+  }
+}
+// 挂载
+mongoose.Schema.Types['File'] = File;
+export default File;

src/entity/meta.ts

@@ -1,5 +1,12 @@
 'use strict';
-
+/**
+ * 表的meta字段
+ * {
+ *  state:{ type: Number, default: 0 },
+ *  createdAt: timestamps,
+ *  updatedAt: timestamps
+ * }
+ */
 export default schema => {
   schema.add({
     meta: {

src/entity/useFile.entity.ts

@@ -0,0 +1,11 @@
+import { modelOptions, prop } from '@typegoose/typegoose';
+/**
+ * 文件地址登记表
+ */
+@modelOptions({
+  schemaOptions: { collection: 'useFile' },
+})
+export class UseFile {
+  @prop({ required: true })
+  uri: string;
+}

src/error/service.error.ts

@@ -1,5 +1,5 @@
 import { MidwayError, registerErrorCode } from '@midwayjs/core';
-
+/**自定义错误编码 */
 export enum ErrorCode {
   /** 未知错误 */
   UNKNOWN = '-1',
@@ -39,6 +39,7 @@ export const FrameworkErrorEnum = registerErrorCode('ServiceError', ErrorCode);
  * @constructor 自定义异常类
  * @param {string} str 异常说明
  * @param {ErrorCode} errcode 错误代码
+ * @param {any} detail 错误细节
  */
 export class ServiceError extends MidwayError {
   constructor(str: string, errcode: string = ErrorCode.UNKNOWN, detail?: any) {

src/index.ts

@@ -23,7 +23,11 @@ export { GetModel } from './util/getModel';
 /**数据库业务服务 */
 export { TransactionService } from './util/transactions';
 /**文件上传 */
-export { FileController } from './controller/file.controller';
-export { FileService } from './service/file.service';
+export { FileController } from './controller/File.controller';
+export { FileService } from './service/File.service';
 /**service工具 */
 export { PageOptions, ResultOptions } from './service/options';
+/**文件处理定时任务队列 */
+export { FileClearProcessor } from './queue/FileClear.queue';
+/**框架默认设置 */
+export * as defaultConfig from './config/config.default'

src/interface/SearchBase.ts

@@ -4,6 +4,8 @@ import { cloneDeep, omit, get, head, last } from 'lodash'
  * @constructor
  * 通过object传过来,名称需要匹配Param
  * @param {Array} like_prop 范围查询字段数组
+ * @param {Array} props 需要处理的属性
+ * @param {object} mapping 属性的映射
  */
 export class SearchBase {
   constructor({ like_prop = [], props = [], mapping = {} }) {

src/interface/VOBase.ts

@@ -1,4 +1,7 @@
 import { get } from 'lodash'
+/**
+ * 输出至视图的格式化基类
+ */
 export class VOBase {
   constructor(response: object) {
     this.errcode = get(response, 'errcode', 0);

src/middleware/response.middleware.ts

@@ -3,6 +3,11 @@ import { Middleware, App } from '@midwayjs/decorator';
 import { NextFunction, Context, Application } from '@midwayjs/koa';
 import { VOBase } from '../interface/VOBase';
 import { get } from 'lodash';
+/**
+ * 返回结果处理拦截器
+ * 响应头的content-type含有'/files'的情况为请求文件,重写响应头即可
+ * 正常请求都要通过视图处理基类(VOBase)格式化后返回
+ */
 @Middleware()
 export class ResponseMiddleware implements IMiddleware<Context, NextFunction> {
   @App()

src/queue/FileClear.queue.ts

@@ -0,0 +1,28 @@
+import { Processor, IProcessor } from '@midwayjs/bull';
+import { FORMAT, Inject } from '@midwayjs/core';
+import { FileService } from '../service/File.service';
+
+/**
+ * 定时任务队列,处理未使用的文件
+ */
+@Processor('FileClear', {
+  repeat: {
+    cron: FORMAT.CRONTAB.EVERY_DAY,
+  },
+})
+export class FileClearProcessor implements IProcessor {
+  @Inject()
+  fileService: FileService;
+
+  async execute() {
+    try {
+      // 先重写文件地址表
+      const modelResults = this.fileService.scanModelHaveFile();
+      await this.fileService.setFileUriFromDefaultDataBase(modelResults);
+      // 再清除未使用的上传文件
+      await this.fileService.deleteNoUseFile();
+    } catch (error) {
+      console.error(error);
+    }
+  }
+}

src/service/file.service.ts

@@ -1,14 +1,175 @@
 import { Config, Inject, Provide } from '@midwayjs/decorator';
-import { existsSync, mkdirSync, renameSync } from 'fs';
+import { existsSync, lstatSync, mkdirSync, readdirSync, renameSync, unlinkSync } from 'fs';
 import { Context } from '@midwayjs/koa';
-import { dirname, extname, sep } from 'path';
+import { dirname, extname, join, sep } from 'path';
+import { ReturnModelType, mongoose } from '@typegoose/typegoose';
+import { isObject, get, pick, flattenDeep } from 'lodash';
+import { GetModel } from '../util/getModel';
+import { InjectEntityModel } from '@midwayjs/typegoose';
+import { UseFile } from '../entity/useFile.entity';
+interface ScanModelFileValue {
+  model: any;
+  projection: object;
+}
+interface ScanModelFileResult {
+  [key: string]: ScanModelFileValue;
+}
+const limit = 50;
+
+/**
+ * 文件上传相关服务
+ */
 @Provide()
 export class FileService {
   @Inject()
   ctx: Context;
   @Config('koa.globalPrefix')
   globalPrefix: string;
+  @Config('upload')
+  uploadConfig;
+  @InjectEntityModel(UseFile)
+  UseFileModel: ReturnModelType<typeof UseFile>;
+
+  // #region 递归清理未被使用的上传文件
+  /**
+   * 删除不在文件使用表登记的文件
+   */
+  async deleteNoUseFile() {
+    const realDir = this.uploadConfig.realdir;
+    this.recursionFindFile(realDir);
+  }
+  /**
+   *
+   * @param basePath 基础路径
+   * @param list 基础路径下的所有内容
+   */
+  async recursionFindFile(basePath) {
+    const dirExists = existsSync(basePath);
+    if (!dirExists) return;
+    const list = readdirSync(basePath);
+    for (const f of list) {
+      const thisPath = join(basePath, f);
+      // 文件夹就继续递归找
+      if (this.isDir(thisPath)) this.recursionFindFile(thisPath);
+      else if (this.isFile(thisPath)) {
+        // 文件,需要到表里找是否存在,存在保留,不存在就删除
+        const shortPath = this.realPathTurnToShortPath(thisPath);
+        const count = await this.UseFileModel.count({ uri: shortPath });
+        if (count <= 0) this.toUnlink(thisPath);
+      }
+    }
+  }
+  /**
+   * 真实路径=>短地址
+   * @param realPath 文件真实路径
+   * @returns string 短地址
+   */
+  realPathTurnToShortPath(realPath: string) {
+    const realDir = this.uploadConfig.realdir;
+    let shortPath = realPath.replace(realDir, `${this.globalPrefix}/files`);
+    while (shortPath.includes('\\')) {
+      shortPath = shortPath.replace('\\', '/');
+    }
+    return shortPath;
+  }
+
+  /**
+   * 删除文件
+   * @param path 文件路径
+   */
+  toUnlink(path) {
+    unlinkSync(path);
+  }
+
+  /**
+   * 判断路径是否存在
+   * @param path 路径
+   * @returns boolean: true-存在/false-不存在
+   */
+  isExists(path) {
+    return existsSync(path);
+  }
+  /**
+   * 判断是否是文件夹
+   * @param path 路径
+   * @returns boolean: true-是文件夹
+   */
+  isDir(path) {
+    const f = lstatSync(path);
+    return f.isDirectory();
+  }
+  /**
+   * 判断是否是文件
+   * @param path 路径
+   * @returns boolean: true-是文件
+   */
+  isFile(path) {
+    const f = lstatSync(path);
+    return f.isFile();
+  }
+  // #endregion
+
+  // #region 写入文件统计地址表
+  /**文件统计地址表重写, 由 scanModelHaveFile()提供参数*/
+  async setFileUriFromDefaultDataBase(scanResult: ScanModelFileResult) {
+    // 清空表内容,重写
+    await this.UseFileModel.deleteMany({});
+    for (const mn in scanResult) {
+      const obj = scanResult[mn];
+      const { model, projection } = obj;
+      // 获取表总数量
+      const total = await model.count();
+      // 计算循环次数
+      const skip = 0;
+      const times = Math.ceil(total / limit);
+      for (let i = 0; i < times; i++) {
+        // 查数据
+        const r = await model.find({}, projection).skip(skip).limit(limit).lean();
+        // 整理批量添加数据格式
+        let filesUri = r.map(i => {
+          const arr = [];
+          for (const prop in projection) {
+            const targetVal = i[prop];
+            const uris = targetVal.map(ti => pick(ti, 'uri'));
+            arr.push(...uris);
+          }
+          return arr;
+        });
+        filesUri = flattenDeep(filesUri);
+        //批量添加
+        await this.UseFileModel.insertMany(filesUri);
+      }
+    }
+  }
+
+  /**扫描注册的model中是否有File类型字段,以Object的形式提取出来*/
+  scanModelHaveFile(): ScanModelFileResult {
+    const modelNames = this.uploadConfig.modelNames || [];
+    const models = modelNames.map(i => GetModel(i));
+    const result: ScanModelFileResult = {};
+    for (const model of models) {
+      const s = model['schema'];
+      const fields: object = s['tree'];
+      const projection = {};
+      for (const f in fields) {
+        const field = fields[f];
+        if (isObject(field)) {
+          const type = get(field, 'type');
+          if (type && type === mongoose.Schema.Types['File']) {
+            projection[f] = 1;
+          }
+        }
+      }
+      if (Object.keys(projection).length > 0) {
+        result[model.modelName] = { model, projection };
+      }
+    }
+    return result;
+  }
+  // #endregion
 
+  // #region 上传部分
+  /**文件真实短地址 */
   getFileShortRealPath() {
     let originalUrl = this.ctx.request.originalUrl;
     //先去掉请求前缀
@@ -60,4 +221,5 @@ export class FileService {
     const secondstr = second < 10 ? '0' + second : second;
     return `${y}${mstr}${dstr}${hstr}${minutestr}${secondstr}`;
   }
+  // #endregion
 }

src/util/getModel.ts

@@ -1,12 +1,17 @@
 import { getModelForClass, getClass } from '@typegoose/typegoose';
 import { AnyParamConstructor } from '@typegoose/typegoose/lib/types';
 import { FrameworkErrorEnum, ServiceError } from '../error/service.error';
-import _ = require('lodash');
+import { upperFirst } from 'lodash';
+/**
+ * 根据表名返回model
+ * @param modelName 表名
+ * @returns model
+ */
 export function GetModel(modelName) {
   let model;
   try {
     model = getModelForClass(
-      getClass(_.upperFirst(modelName)) as AnyParamConstructor<any>
+      getClass(upperFirst(modelName)) as AnyParamConstructor<any>
     );
   } catch (error) {
     throw new ServiceError(`${modelName}-生成模型错误`, FrameworkErrorEnum.SERVICE_FAULT);

src/util/transactions.ts

@@ -37,7 +37,9 @@ interface IMission {
   /**任务范围(添加不需要) */
   query?: object;
 }
-
+/**
+ * 事务服务工具类
+ */
 @Provide()
 export class TransactionService {
   mission: Array<IMission> = [];