导出数据量太大,分割做,先做的问卷

app/service/questionnaire.js

@@ -1,6 +1,5 @@
 'use strict';
 
-
 const assert = require('assert');
 const _ = require('lodash');
 const moment = require('moment');
@@ -79,7 +78,9 @@ class QuestionnaireService extends CrudService {
   // 建立任务
   async toExport(body) {
     const { range, questionnaireid } = body;
-    const questionnaire = await this.questionnairemodel.findById(questionnaireid);
+    const questionnaire = await this.questionnairemodel.findById(
+      questionnaireid
+    );
     if (!questionnaire) {
       throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到问卷信息');
     }
@@ -132,66 +133,60 @@ class QuestionnaireService extends CrudService {
         return i;
       });
       // 获取问卷
-      let questionnaire = await this.questionnairemodel.findById(questionnaireid);
+      const questionnaire = await this.questionnairemodel.findOne(
+        { _id: ObjectId(questionnaireid) },
+        'name'
+      );
       if (!questionnaire) {
         throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到问卷信息');
       }
-      this.ctx.service.util.updateProcess(missionid, '25');
-      questionnaire = JSON.parse(JSON.stringify(questionnaire));
+      let fn = await this.toSetFileName(questionnaire.name, range);
+      // this.ctx.service.util.updateProcess(missionid, '25');
       // 修改条件,termid变成数组了,需要一个一个查出来
-      const { planid, termid, batchid, classid } = range;
+      const { planid, batchid, classid } = range;
+      let { termid } = range;
       const queryObject = {};
       if (planid) queryObject.planid = planid;
       if (batchid) queryObject.batchid = batchid;
       if (classid) queryObject.classid = classid;
-      const studentList = [];
-      const questAnswerList = [];
+      let count = 0;
+      let dp;
+      const qkeys = Object.keys(range);
+      if (qkeys.length === 2 && termid.length <= 0) {
+        // 说明只有planid
+        const plan = await this.ctx.model.Trainplan.findOne({ _id: ObjectId(planid) });
+        if (!plan) throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到指定年度计划');
+        termid = plan.termnum.map(i => JSON.parse(JSON.stringify(i._id)));
+      }
       for (const t of termid) {
         queryObject.termid = t;
-        const obj = await this.toGetData(queryObject, questionnaireid);
-        if (!obj) continue;
-        const { studentList: stuList, questAnswerList: qaList } = obj;
-        if (stuList) studentList.push(...stuList);
-        if (qaList) questAnswerList.push(...qaList);
-      }
-      this.ctx.service.util.updateProcess(missionid, '50');
-      // fn,根据范围+问卷 得出文件名
-      const fn = await this.toSetFileName(questionnaire.name, range);
-
-      let excelData;
-      if (direction === 'horizontal') {
-        excelData = this.horizontalSetData(
-          studentList,
-          questAnswerList,
+        let data = await this.toGetData(
+          queryObject,
+          questionnaireid,
+          direction,
           modelList
         );
-      } else if (direction === 'vertical') {
-        excelData = this.verticaSetData(
-          studentList,
-          questAnswerList,
-          modelList
-        );
-      } else {
-        throw new BusinessError(
-          ErrorCode.DATA_NOT_EXIST,
-          '未找到excel的表头排列方式'
-        );
-      }
-      this.ctx.service.util.updateProcess(missionid, '75');
-      if (excelData) {
-        const { head, data } = excelData;
-        const res = await this.ctx.service.util.toExcel(data, head, fn);
-        if (!res) {
-          console.error(
-            `${moment().format('YYYY-MM-DD HH:SS:mm')} ${fn} 导出失败`
-          );
-          throw new BusinessError(ErrorCode.SERVICE_FAULT, `${fn}导出失败`);
+        if (!data) continue;
+        if (count !== 0 && direction === 'horizontal') data.shift();
+        if (count !== 0 && direction === 'vertical') {
+          data = data.map(i => {
+            i.shift();
+            return i;
+          });
         }
-        this.ctx.service.util.updateProcess(missionid, '100', '2', {
-          uri: res,
-        });
+        const {
+          downloadPath,
+          fn: newFileName,
+        } = await this.ctx.service.util.toAsyncExcel(data, fn, dp, direction);
+        dp = downloadPath;
+        fn = newFileName;
+        count++;
       }
+      this.ctx.service.util.updateProcess(missionid, '100', '2', {
+        uri: dp,
+      });
     } catch (error) {
+      console.log(error);
       this.ctx.service.util.updateProcess(missionid, undefined, '3');
     }
   }
@@ -200,27 +195,52 @@ class QuestionnaireService extends CrudService {
    * 获取学生与学生回答的问卷
    * @param {Object} condition 查询学生和学生回答问卷的条件
    * @param {String} questionnaireid 问卷id
+   * @param {String} direction 方向
+   * @param {Array} modelList 导出字段
    */
-  async toGetData(condition, questionnaireid) {
+  async toGetData(condition, questionnaireid, direction, modelList) {
     // 获取学生
     let studentList = await this.ctx.service.student.query(condition);
     if (studentList.length <= 0) {
-      throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到任何学生信息');
+      this.ctx.logger.error(`${JSON.stringify(condition)}条件下,未找到任何信息`);
+      return;
+      // throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到任何学生信息');
     }
     studentList = JSON.parse(JSON.stringify(studentList));
     // 再获取问卷答案
-    let questAnswerList = await this.ctx.model.Uploadquestion.find({
+    const questAnswerList = await this.ctx.model.Uploadquestion.find({
       ...condition,
       questionnaireid,
     });
     if (!questAnswerList) {
-      throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到任何完成的问卷');
+      this.ctx.logger.error(`${JSON.stringify(condition)}&${questionnaireid}条件下,未找到任何完成的问卷`);
+      return;
     }
-    questAnswerList = JSON.parse(JSON.stringify(questAnswerList));
-    const obj = {};
-    if (studentList) obj.studentList = studentList;
-    if (questAnswerList) obj.questAnswerList = questAnswerList;
-    return obj;
+    let res = [];
+    let data = [];
+    if (direction === 'horizontal') {
+      const head = this.getHead(direction, modelList, studentList.length);
+      data = this.horizontalSetData(studentList, questAnswerList, modelList);
+      res.push(head.map(i => i.header));
+      for (const d of data) {
+        const mid = [];
+        for (const h of head) {
+          const { key } = h;
+          mid.push(_.get(d, key, ''));
+        }
+        res.push(mid);
+      }
+
+    } else if (direction === 'vertical') {
+      data = this.verticaSetData(studentList, questAnswerList, modelList);
+      res = data.map(i => Object.values(i));
+    } else {
+      throw new BusinessError(
+        ErrorCode.DATA_NOT_EXIST,
+        '未找到excel的表头排列方式'
+      );
+    }
+    return res;
   }
 
   /**
@@ -291,7 +311,9 @@ class QuestionnaireService extends CrudService {
     const trainPlan = await this.ctx.model.Trainplan.findOne({
       'termnum._id': ObjectId(termid),
     });
-    if (!trainPlan) { throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到年度计划信息'); }
+    if (!trainPlan) {
+      throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找到年度计划信息');
+    }
     const { termnum } = trainPlan;
     if (!termnum) {
       throw new BusinessError(
@@ -312,6 +334,34 @@ class QuestionnaireService extends CrudService {
     return obj;
   }
 
+  /**
+   * 取出对应方向的表头
+   * @param {String} type 横纵向类型
+   * @param {Array} modelList 导出字段
+   * @param {Number} studentList count的学生数量
+   */
+  getHead(type, modelList, studentList) {
+    let head = [];
+    if (type === 'horizontal') {
+      // 横向头
+      head = modelList.map(i => {
+        const { zh, model, _id } = i;
+        const headObj = { header: zh };
+        if (model) headObj.key = model;
+        else if (_id) headObj.key = _id;
+        headObj.width = 20;
+        return headObj;
+      });
+    } else {
+      // 纵向头
+      const head = [{ key: 'c', width: 20 }];
+      for (let i = 1; i <= studentList.length; i++) {
+        head.push({ key: `c${i}`, width: 20 });
+      }
+    }
+    return head;
+  }
+
   /**
    * 横向组织数据
    * @param {Array} studentList 学生列表
@@ -319,16 +369,7 @@ class QuestionnaireService extends CrudService {
    * @param {Array} modelList 需要导出的字段
    */
   horizontalSetData(studentList, questAnswerList, modelList) {
-    // 第一步,组织excel表头
-    const head = modelList.map(i => {
-      const { zh, model, _id } = i;
-      const headObj = { header: zh };
-      if (model) headObj.key = model;
-      else if (_id) headObj.key = _id;
-      headObj.width = 20;
-      return headObj;
-    });
-    // 第二步,组织数据需要按照modelList的顺序进行整理
+    // 组织数据需要按照modelList的顺序进行整理
     const data = [];
     for (const stu of studentList) {
       const obj = {};
@@ -355,10 +396,10 @@ class QuestionnaireService extends CrudService {
       }
       data.push(obj);
     }
-    const obj = {};
-    if (head) obj.head = head;
-    if (data) obj.data = data;
-    return obj;
+    // const obj = {};
+    // if (head) obj.head = head;
+    // if (data) obj.data = data;
+    return data;
   }
 
   /**
@@ -368,12 +409,7 @@ class QuestionnaireService extends CrudService {
    * @param {Array} modelList 需要导出的字段
    */
   verticaSetData(studentList, questAnswerList, modelList) {
-    // 第一步,组织excel表头,纵向不需要第一行表头,开始就是数据,表头是第一列
-    const head = [{ key: 'c', width: 20 }];
-    for (let i = 1; i <= studentList.length; i++) {
-      head.push({ key: `c${i}`, width: 20 });
-    }
-    // 第二部,整理数据
+    // 整理数据
     const data = [];
     for (const m of modelList) {
       const { table, model, _id, zh } = m;
@@ -404,10 +440,7 @@ class QuestionnaireService extends CrudService {
       }
       data.push(obj);
     }
-    const obj = {};
-    if (head) obj.head = head;
-    if (data) obj.data = data;
-    return obj;
+    return data;
   }
 }
 

app/service/student.js

@@ -83,42 +83,31 @@ class StudentService extends CrudService {
           model: 'Class',
           select: 'name',
         },
-        {
-          path: 'planid',
-          model: 'Trainplan',
-        },
       ])
       .skip(parseInt(skip))
       .limit(parseInt(limit));
+    const planids = _.uniq(data.map(i => ObjectId(i.planid)));
+    const plan = await this.ctx.model.Trainplan.find({ _id: planids });
     data = JSON.parse(JSON.stringify(data));
-    for (const stu of data) {
-      const { classid, planid, termid, batchid } = stu;
-      // 先拿学生
-      if (classid && _.isObject(classid)) {
-        const { _id, name } = classid;
-        if (name) stu.classname = name;
-        if (_id) stu.classid = _id;
+    data = data.map(i => {
+      if (i.classid && _.isObject(i.classid)) {
+        i.classname = _.get(i.classid, 'name');
+        i.classid = _.get(i.classid, '_id');
       }
-      // 拼计划信息
-      if (planid && _.isObject(planid)) {
-        const { termnum, _id } = planid;
-        if (termnum && _.isArray(termnum)) {
-          const t = termnum.find(f => f._id === termid);
-          if (t) {
-            const { batchnum, term } = t;
-            if (term) stu.termname = term;
-            if (batchnum && _.isArray(batchnum)) {
-              const b = batchnum.find(f => f._id === batchid);
-              if (b) {
-                const { batch } = b;
-                if (batch) stu.batchname = batch;
-              }
-            }
+      const p = plan.find(f => ObjectId(f._id).equals(i.planid));
+      if (p) {
+        const { termnum } = p;
+        const tr = termnum.id(i.termid);
+        if (tr) {
+          i.termname = _.get(tr, 'term');
+          const br = tr.batchnum.id(i.batchid);
+          if (br) {
+            i.batchname = _.get(br, 'batch');
           }
         }
-        stu.planid = _id;
       }
-    }
+      return i;
+    });
     return data;
   }
 
@@ -598,11 +587,17 @@ class StudentService extends CrudService {
         else queryObject.isComming = isComming;
       }
       let studentList = [];
-      for (const t of termid) {
-        queryObject.termid = t;
-        const stuList = await this.query(queryObject);
-        studentList.push(...stuList);
+      console.log(queryObject);
+      if (termid && termid.length > 0) {
+        for (const t of termid) {
+          queryObject.termid = t;
+          const stuList = await this.query(queryObject);
+          studentList.push(...stuList);
+        }
+      } else {
+        studentList = await this.query(queryObject);
       }
+      console.log(studentList.length);
       // 获取学生,4分之1
       this.ctx.service.util.updateProcess(missionid, '25');
       console.log('学生导出=>25%');
@@ -633,6 +628,7 @@ class StudentService extends CrudService {
       console.log('学生导出=>50%');
       let fn = '学生名单';
       // 因为是递进下来, batchid和classid并列,并非递进
+      // 获取fn
       if (classid) {
         // 文件名称: 期, 班
         // 班级名上面有直接拽来

app/service/util.js

@@ -217,6 +217,63 @@ class UtilService extends CrudService {
     return `/files/excel/${filename}`;
   }
 
+  /**
+   * 创建/重复写入excel
+   * @param {Array} dataList 数据
+   * @param {String} fn 文件名, 第一次进入时,只是单纯的名,第二次开始,有后缀与时间戳
+   * @param {String} downloadPath 下载文件路径
+   * @param {String} type 方向,没有默认横向
+   */
+  async toAsyncExcel(dataList, fn = '导出结果', downloadPath, type) {
+    const { app } = this;
+    const workbook = new Excel.Workbook();
+    let sheet;
+    // 取出预设存储地址
+    const rootPath = `${app.config.cdn.repos_root_path}`;
+    const rooturl = `${app.config.cdn.repos_root_url_excel}`;
+    let path = `${rootPath}${rooturl}`;
+    let filepath = '';
+
+    if (!path) {
+      throw new BusinessError(ErrorCode.BUSINESS, '服务端没有设置存储路径');
+    }
+    if (process.env.NODE_ENV === 'development') path = 'E:\\exportFile\\excel\\';
+    if (!fs.existsSync(path)) {
+      // 如果不存在文件夹,就创建
+      fs.mkdirSync(path);
+    }
+
+    if (!downloadPath) {
+      // 第一次进入,文件还未生成
+      const nowDate = new Date().getTime();
+      fn = `${fn}-${nowDate}.xlsx`;
+      sheet = workbook.addWorksheet('sheet');
+    } else {
+      let domain = 'http://127.0.0.1';
+      if (process.env.NODE_ENV === 'development')domain = `${domain}:8000`;
+      const file = await this.ctx.curl(`${domain}${downloadPath}`);
+      if (!(file && file.data)) {
+        throw new BusinessError(ErrorCode.DATA_NOT_EXIST, '未找导出的excel');
+      }
+      // 读取文件
+      await workbook.xlsx.load(file.data);
+      sheet = workbook.getWorksheet('sheet');
+    }
+    if (!type) sheet.addRows(dataList);
+    else {
+      for (let i = 1; i <= dataList.length; i++) {
+        const element = dataList[i - 1];
+        const rows = sheet.getRow(i);
+        if (rows.values.length <= 0) rows.values = element;
+        else rows.values = rows.values.concat(element);
+        rows.commit();
+      }
+    }
+    filepath = `${path}${fn}`;
+    await workbook.xlsx.writeFile(filepath);
+    return { downloadPath: `/files/excel/${fn}`, fn };
+  }
+
   /**
    * 导出docx
    * @param {Array} data 数据[{title,content([]),author}]