1.添加任务日志

2.添加node和mongodb连接的超时时间为900s
3.调整任务顺序

app/schedule/updateData.js

@@ -9,7 +9,7 @@ class UpdateData extends Subscription {
       // interval: '24h', // 1 分钟间隔
       cron: '0 0 5 * * ? ', // 每天凌晨6点
       type: 'worker', //  all 指定所有的都需要执行  worker 其中一个
-      // immediate: true,
+      immediate: true,
       cronOptions: {
         tz: 'Asia/Shanghai',
       },

app/service/onlineUserService2.js

@@ -153,10 +153,13 @@ class OnlineUserService2 extends Service {
 
     const { appTotal, iviTotal, tBoxTotal } = await this.count(timeRangData);
     const total = appTotal + tBoxTotal;
+    ctx.logger.info('任务进行count');
 
     const { faceTotal, voiceTotal, pwdTotal, codeTotal, scanTotal } = await this.modeGroup(timeRangData);
+    ctx.logger.info('任务进行modeGroup');
 
     const { carTotal, commonTotal, guestTotal } = await this.roleGroup(timeRangData);
+    ctx.logger.info('任务进行roleGroup');
 
     let successTotal = 0;
     let failTotal = 0;
@@ -173,35 +176,46 @@ class OnlineUserService2 extends Service {
           break;
       }
     });
+    ctx.logger.info('任务进行statusGroup');
 
     const aResult = await this.appPlace(timeRangData);
     const appPlace = aResult.areas || [];
+    ctx.logger.info('任务进行appPlace');
 
     const tResult = await this.tBoxPlace(timeRangData);
     const tBoxPlace = tResult.areas || [];
+    ctx.logger.info('任务进行tBoxPlace');
 
 
     const al = await this.appLocation(timeRangData);
     const appLocation = al.areas || [];
+    ctx.logger.info('任务进行appLocation');
 
     const anl = await this.appNewLocation(timeRangData);
     const appNewLocation = anl.areas || [];
+    ctx.logger.info('任务进行appNewLocation');
 
 
     const aal = await this.appActiveLocation(timeRangData);
     const appActiveLocation = aal.areas || [];
+    ctx.logger.info('任务进行appActiveLocation');
 
     const il = await this.iviLocation(timeRangData);
     const iviLocation = il.areas || [];
+    ctx.logger.info('任务进行iviLocation');
 
     const inl = await this.iviNewLocation(timeRangData);
     const iviNewLocation = inl.areas || [];
+    ctx.logger.info('任务进行iviNewLocation');
 
     const ial = await this.iviActiveLocation(timeRangData);
     const iviActiveLocation = ial.areas || [];
+    ctx.logger.info('任务进行iviActiveLocation');
 
     const appYear = await this.getAppYearOnline(timeRangData);
+    ctx.logger.info('任务进行getAppYearOnline');
     const tBoxYear = await this.getTboxYearOnline(timeRangData);
+    ctx.logger.info('任务进行getTboxYearOnline');
 
     await ctx.service.statisticsService.save(ctx.model.Local.OnlineUserModel,
       { ...initData, total, appTotal, tBoxTotal, appPlace, tBoxPlace,

app/service/statisticsService.js

@@ -41,19 +41,6 @@ class StatisticsService extends Service {
   async taskCond(baseData) {
     const { ctx } = this;
     // ?
-    await this.sign([ 'app_task', 'ivi_online_user', 't_rbac_user' ], baseData, async () => {
-      try {
-        await ctx.service.tRbacUserService.statistics(baseData);// 重新处理
-      } catch (e) {
-        await this.writeError(baseData, e, 'tRbacUserService');
-      }
-      try {
-        await ctx.service.onlineUserService2.statistics(baseData);// 重新处理
-      } catch (e) {
-        await this.writeError(baseData, e, 'onlineUserService2');
-      }
-    });
-
     await this.sign([ 'app_task' ], baseData, async () => {
       try {
         await ctx.service.appBehaviorRecordService.statistics(baseData);
@@ -71,47 +58,61 @@ class StatisticsService extends Service {
         await this.writeError(baseData, e, 'appFeedbackRecordService');
       }
     });
-    await this.sign([ 'driving_behavior_info', 't_vehicle_record' ], baseData, async () => {
+    await this.sign([ 'msc_query_record' ], baseData, async () => {
       try {
-        await ctx.service.tVehicleRecordService2.statistics(baseData);// 重新处理
+        await ctx.service.msgQueryRecordService.statistics(baseData);
       } catch (e) {
-        await this.writeError(baseData, e, 'tVehicleRecordService2');
+        await this.writeError(baseData, e, 'msgQueryRecordService');
       }
+    });
+    await this.sign([ 'msg_center_record' ], baseData, async () => {
       try {
-        await ctx.service.tVehicleReportInfoService.statistics(baseData);// 重新处理
+        await ctx.service.msgCenterRecordService2.statistics(baseData);
       } catch (e) {
-        await this.writeError(baseData, e, 'tVehicleReportInfoService');
+        await this.writeError(baseData, e, 'msgCenterRecordService2');
       }
+    });
+    await ctx.service.tBoxAutoTestService.statistics(baseData);// 重新处理
+    await this.sign([ 'stats_base_info', 't_vehicle_record' ], baseData, async () => {
       try {
-        await ctx.service.drivingBehaviorInfoService2.statistics(baseData);// 重新处理
+        await ctx.service.statsBaseInfoService2.statistics(baseData);// 重新处理
       } catch (e) {
-        await this.writeError(baseData, e, 'drivingBehaviorInfoService2');
+        await this.writeError(baseData, e, 'statsBaseInfoService2');
+      }
+    });
+
+    await this.sign([ 'app_task', 'ivi_online_user', 't_rbac_user' ], baseData, async () => {
+      try {
+        await ctx.service.tRbacUserService.statistics(baseData);// 重新处理
+      } catch (e) {
+        await this.writeError(baseData, e, 'tRbacUserService');
       }
       try {
-        await ctx.service.tFsmHitoryService.statistics(baseData);// 重新处理
+        await ctx.service.onlineUserService2.statistics(baseData);// 重新处理
       } catch (e) {
-        await this.writeError(baseData, e, 'tFsmHitoryService');
+        await this.writeError(baseData, e, 'onlineUserService2');
       }
     });
-    await this.sign([ 'stats_base_info', 't_vehicle_record' ], baseData, async () => {
+    await this.sign([ 'driving_behavior_info', 't_vehicle_record' ], baseData, async () => {
       try {
-        await ctx.service.statsBaseInfoService2.statistics(baseData);// 重新处理
+        await ctx.service.tVehicleRecordService2.statistics(baseData);// 重新处理
       } catch (e) {
-        await this.writeError(baseData, e, 'statsBaseInfoService2');
+        await this.writeError(baseData, e, 'tVehicleRecordService2');
       }
-    });
-    await this.sign([ 'msc_query_record' ], baseData, async () => {
       try {
-        await ctx.service.msgQueryRecordService.statistics(baseData);
+        await ctx.service.tVehicleReportInfoService.statistics(baseData);// 重新处理
       } catch (e) {
-        await this.writeError(baseData, e, 'msgQueryRecordService');
+        await this.writeError(baseData, e, 'tVehicleReportInfoService');
       }
-    });
-    await this.sign([ 'msg_center_record' ], baseData, async () => {
       try {
-        await ctx.service.msgCenterRecordService2.statistics(baseData);
+        await ctx.service.drivingBehaviorInfoService2.statistics(baseData);// 重新处理
       } catch (e) {
-        await this.writeError(baseData, e, 'msgCenterRecordService2');
+        await this.writeError(baseData, e, 'drivingBehaviorInfoService2');
+      }
+      try {
+        await ctx.service.tFsmHitoryService.statistics(baseData);// 重新处理
+      } catch (e) {
+        await this.writeError(baseData, e, 'tFsmHitoryService');
       }
     });
     if (baseData.timeRangData.startTime >= ctx.helper.timeMonthSeven) {
@@ -123,7 +124,6 @@ class StatisticsService extends Service {
         }
       });
     }
-    await ctx.service.tBoxAutoTestService.statistics(baseData);// 重新处理
   }
 
 
@@ -160,7 +160,7 @@ class StatisticsService extends Service {
     }
     if (!satrtMoment) {
       satrtMoment = moment(this.ctx.helper.yesterday());
-      // satrtMoment = moment('2021-06-01');
+      // satrtMoment = moment('2020-01-01');
     }
     endTime = endTime || this.ctx.helper.today();
     ctx.logger.info(`任务进行 开始${satrtMoment.valueOf()} 结束${endTime}`);

app/service/tRbacUserService.js

@@ -37,54 +37,75 @@ class TRbacUserService extends Service {
     }
     initData.start_time = new Date();
     const total = await this.total({ create_date: { $lt: timeRangData.endTime } });
+    ctx.logger.info('任务进行total');
     const newTotal = await this.total({ create_date:
           { $gte: timeRangData.startTime, $lt: timeRangData.endTime } });
+    ctx.logger.info('任务进行newTotal');
     const sexAndAgeTotal = await this.sex2Age({ create_date: { $lt: timeRangData.endTime } });
+    ctx.logger.info('任务进行sexAndAgeTotal');
     let appTotal = 0;
     if (timeRangData.startTime >= ctx.helper.timeMonthSeven) {
       appTotal = await this.total({
         create_date: { $lt: timeRangData.endTime }, user_orgin: 'APP',
       });
+      ctx.logger.info('任务进行appTotal');
     }
     const appSexAndAgeTotal = await this.sex2Age({
       create_date: { $lt: timeRangData.endTime }, user_orgin: 'APP' });
+    ctx.logger.info('任务进行appSexAndAgeTotal');
     const iviTotal = await this.total({
       create_date: { $lt: timeRangData.endTime }, user_orgin: 'IVI' });
+    ctx.logger.info('任务进行iviTotal');
     const iviSexAndAgeTotal = await this.sex2Age({
       create_date: { $lt: timeRangData.endTime }, user_orgin: 'IVI' });
+    ctx.logger.info('任务进行iviSexAndAgeTotal');
     let newAppTotal = 0;
     if (timeRangData.startTime >= ctx.helper.timeMonthSeven) {
       newAppTotal = await this.total({
         create_date: { $gte: timeRangData.startTime, $lt: timeRangData.endTime }, user_orgin: 'APP',
       });
+      ctx.logger.info('任务进行newAppTotal');
     }
     const newAppSexAndAgeTotal = await this.sex2Age({
       create_date: { $gte: timeRangData.startTime, $lt: timeRangData.endTime }, user_orgin: 'APP' });
+    ctx.logger.info('任务进行newAppSexAndAgeTotal');
     const newIviTotal = await this.total({
       create_date: { $gte: timeRangData.startTime, $lt: timeRangData.endTime }, user_orgin: 'IVI' });
+    ctx.logger.info('任务进行newIviTotal');
 
     const newIviSexAndAgeTotal = await this.sex2Age({
       create_date: { $gte: timeRangData.startTime, $lt: timeRangData.endTime }, user_orgin: 'IVI' });
+    ctx.logger.info('任务进行newIviSexAndAgeTotal');
     let activeAppTotal = 0;
     if (timeRangData.startTime >= ctx.helper.timeMonthSeven) {
       activeAppTotal = await this.activeAppTotal(timeRangData);
+      ctx.logger.info('任务进行activeAppTotal');
     }
     const activeAppMonth = await this.activeAppTotal2({ startTime: ctx.helper.getMonthTop(timeRangData.startTime),
       endTime: timeRangData.endTime }, 3);
+    ctx.logger.info('任务进行activeAppMonth');
     const activeAppYear = await this.activeAppTotal2({ startTime: ctx.helper.nowYear(timeRangData.startTime),
       endTime: timeRangData.endTime }, 30);
+    ctx.logger.info('任务进行activeAppYear');
     const activeAppSexAndAgeTotal = await this.activeAppSex2Age(timeRangData);
+    ctx.logger.info('任务进行activeAppSexAndAgeTotal');
     const activeIviTotal = await this.activeIviTotal(timeRangData);
+    ctx.logger.info('任务进行activeIviTotal');
     const activeIviMonth = await this.activeIviTotal2({ startTime: ctx.helper.getMonthTop(timeRangData.startTime),
       endTime: timeRangData.endTime }, 3);
+    ctx.logger.info('任务进行activeIviMonth');
     const activeIviYear = await this.activeIviTotal2({ startTime: ctx.helper.nowYear(timeRangData.startTime),
       endTime: timeRangData.endTime }, 30);
+    ctx.logger.info('任务进行activeIviYear');
     const activeIviSexAndAgeTotal = await this.activeIviSex2Age(timeRangData);
-
+    ctx.logger.info('任务进行activeIviSexAndAgeTotal');
     // 目前的所有性别年龄统计 和这下面的3个 算法-算车主都是算最终态
     const saledTotal = await this.total({ role_id: ctx.helper.saledRoleId });
+    ctx.logger.info('任务进行saledTotal');
     const saledAppTotal = await this.total({ user_orgin: 'APP', role_id: ctx.helper.saledRoleId });
+    ctx.logger.info('任务进行saledAppTotal');
     const saledIviTotal = await this.total({ user_orgin: 'IVI', role_id: ctx.helper.saledRoleId });
+    ctx.logger.info('任务进行saledIviTotal');
     await ctx.service.statisticsService.save(ctx.model.Local.TRbacUserModel,
       { ...initData, total, newTotal, sexAndAgeTotal,
         appTotal, appSexAndAgeTotal,

app/service/tVehicleRecordService2.js

@@ -196,7 +196,7 @@ class TVehicleRecordService2 extends Service {
       } },
     ];
 
-    return await ctx.model.TVehicleRecordModel.aggregate(agg).allowDiskUse(true);;
+    return await ctx.model.TVehicleRecordModel.aggregate(agg).allowDiskUse(true);
   }
 
   // TODO 暂时即时查询
@@ -352,18 +352,22 @@ class TVehicleRecordService2 extends Service {
     }
     initData.start_time = new Date();
     const result = await this.group(timeRangData);
+    ctx.logger.info('任务进行group');
     const { activeVin, aResult } = await this.activeCount(timeRangData);
+    ctx.logger.info('任务进行activeCount');
     const onlineCar = await this.onlineCount(timeRangData);
     const car = result.car || [];
     aResult.forEach(c => {
       const obj = car.find(item => item._id.series_code == c._id.series_code && item._id.model_code == c._id.model_code);
       if (obj) {
-        obj.count = c.count;
         obj.activeCount = c.activeCount;
       }
     });
+    ctx.logger.info('任务进行onlineCount');
     const saledUser = await this.saledUserCount(timeRangData);
+    ctx.logger.info('任务进行saledUserCount');
     const saledNewUser = await this.saledNewUserCount(timeRangData, saledUser);
+    ctx.logger.info('任务进行saledNewUserCount');
 
     await ctx.service.statisticsService.save(ctx.model.Local.TVehicleRecordModel,
       { ...initData, car, onlineCar, saledNewUser, saledUser, activeVin }, isForceUpdate);
@@ -373,9 +377,10 @@ class TVehicleRecordService2 extends Service {
   async group({ startTime, endTime }) {
     const { ctx } = this;
     const agg = [
-      { $match: { sale_date: { $lt: endTime } } },
+      // { $match: { sale_date: { $lt: endTime } } },
       { $group: { _id: { series_code: '$series_code', model_code: '$model_code' },
         series_name: { $first: '$series_name' }, model_name: { $first: '$model_name' },
+        count: { $sum: 1 },
         saledCount: { $sum: {
           $cond: [{ $and: [{ $eq: [ '$is_saled_car', 1 ] }, { $lt: [ '$sale_date', endTime ] },
             { $ne: [{ $ifNull: [ '$user_id', null ] }, null ] }] }, 1, 0 ],
@@ -412,17 +417,19 @@ class TVehicleRecordService2 extends Service {
       { $match: { mileage: { $gt: 50 } } },
     ];
     const result1 = await ctx.model.TBoxOnlineModel.aggregateFix(onlineAgg);
+    ctx.logger.info('任务进行onlineAgg');
     const result2 = await ctx.model.DrivingBehaviorInfoModel.aggregateFix(driveAgg);
+    ctx.logger.info('任务进行driveAgg');
     const unionArray = ctx.helper.unionArray(result1.map(item => item._id), result2.map(item => item._id));
-
+    ctx.logger.info('任务进行unionArray');
     const agg = [
-      { $match: { create_time: { $lt: endTime } } },
+      { $match: { create_time: { $lt: endTime }, vin: { $in: unionArray } } },
       { $group: { _id: { series_code: '$series_code', model_code: '$model_code' },
-        count: { $sum: 1 },
-        activeCount: { $sum: { $cond: [{ $in: [ '$vin', unionArray ] }, 1, 0 ] } },
+        activeCount: { $sum: 1 },
       } },
     ];
     const aResult = await ctx.model.TVehicleRecordModel.aggregateFix(agg);
+    ctx.logger.info('任务进行agg');
     return { activeVin: unionArray, aResult };
   }
 
@@ -432,7 +439,7 @@ class TVehicleRecordService2 extends Service {
     const agg = [
       { $match: ctx.helper.getTimeRangMatch(startTime, endTime, 'online_time') },
       { $group: { _id: '$vin', count: { $sum: 1 } } },
-      { $lookup: { from: 't_vehicle_record', localField: 'vin', foreignField: 'vin', as: 'car' } },
+      { $lookup: { from: 't_vehicle_record', localField: '_id', foreignField: 'vin', as: 'car' } },
       { $unwind: { path: '$car', preserveNullAndEmptyArrays: true } },
       { $group: { _id: { series_code: '$car.series_code', model_code: '$car.model_code' },
         count: { $sum: 1 } } },

config/config.local.js

@@ -22,25 +22,25 @@ module.exports = () => {
       etlDB: { // 原始数据库
         url: 'mongodb://faw-etl:faw-etl123@10.120.9.133:27042,10.120.9.134:27042,10.120.9.135:27042/faw-etl?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlLocalDB: { // 本地清洗后的数据库
         url: 'mongodb://faw-etl:faw-etl123@10.120.9.133:27042,10.120.9.134:27042,10.120.9.135:27042/faw-etl-local3?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlAutoDB: {
         url: 'mongodb://faw:fawl123@10.120.8.56:27041,10.120.8.19:27041,10.120.8.38:27041/faw?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
     },

config/config.prod3.js

@@ -25,25 +25,25 @@ module.exports = () => {
       etlDB: { // 原始数据库
         url: 'mongodb://faw-etl:faw-etl123@10.120.9.133:27042,10.120.9.134:27042,10.120.9.135:27042/faw-etl?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlLocalDB: { // 本地清洗后的数据库
         url: 'mongodb://faw-etl:faw-etl123@10.120.9.133:27042,10.120.9.134:27042,10.120.9.135:27042/faw-etl-local3?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlAutoDB: {
         url: 'mongodb://faw:fawl123@10.120.8.56:27041,10.120.8.19:27041,10.120.8.38:27041/faw?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
     },

config/config.sit.js

@@ -24,25 +24,25 @@ module.exports = () => {
       etlDB: { // 原始数据库
         url: 'mongodb://faw-etl:faw-etl123@uat.dbaas.private:27028/faw-etl', // uat
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlLocalDB: { // 本地清洗后的数据库
         url: 'mongodb://local3:local3%40123@uat.dbaas.private:27028/faw-etl-local3', // uat
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlAutoDB: { // 自动化测试数据库
         url: 'mongodb://admin:mongo123@uat.dbaas.private:27028/faw', // uat
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
     },

config/config.sit3.js

@@ -26,25 +26,25 @@ module.exports = () => {
       etlDB: { // 原始数据库
         url: 'mongodb://faw-etl-admin:faw-etl-admin%40123@10.112.16.11:27017/faw-etl', // sit
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlLocalDB: { // 本地清洗后的数据库
         url: 'mongodb://faw-etl-local:faw-etl-local%40123@10.112.16.11:27017/faw-etl-local', // sit
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlAutoDB: { // 自动化测试数据库
         url: 'mongodb://fawtsp:fawtsp@10.112.16.11:27017/faw', // sit
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
     },

config/config.unittest.js

@@ -25,25 +25,25 @@ module.exports = () => {
       etlDB: { // 原始数据库
         url: 'mongodb://faw-etl:faw-etl123@10.120.9.133:27042,10.120.9.134:27042,10.120.9.135:27042/faw-etl?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlLocalDB: { // 本地清洗后的数据库
         url: 'mongodb://faw-etl:faw-etl123@10.120.9.133:27042,10.120.9.134:27042,10.120.9.135:27042/faw-etl-local3?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
       etlAutoDB: {
         url: 'mongodb://faw:fawl123@10.120.8.56:27041,10.120.8.19:27041,10.120.8.38:27041/faw?replicaSet=rs', // 生产
         options: {
+          socketTimeoutMS: 900000,
           useUnifiedTopology: true,
           poolSize: 40,
-          reconnectTries: 86400,
         },
       },
     },