aiquery2/api/app/controllers/videoRecognitionDemo.js

const qiniu = require('qiniu');
const moment = require('moment');
const uuid = require('uuid');
const superagent = require('superagent');
const { reportBusinessCall, reportFastgptResponse, resolveContextUserId } = require('../services/dashboardReporter');

module.exports.vrdemo = async (ctx, next) => {
   let {
      url,
      base64, fileName, key: keyPath, prompt
   } = ctx.request.body;

   try {
      const { qiniu: { dmn: domain, bkt: bucket, ak: accessKey, sk: secretKey } } = ctx.config;

      if (url) {
         const response = await superagent
            .get(url)
            .responseType('blob')  // 或者 'arraybuffer',主要是为了拿到 buffer
            .buffer(true);         // 强制 superagent 处理成 buffer

         const contentType = response.headers['content-type']; // 获取图片类型
         const buffer = response.body; // 图片数据是 buffer
         base64 = `data:${contentType};base64,` + buffer.toString('base64');
      }


      if (!base64) {
         ctx.body = { code: 400, message: '缺少base64参数' };
         return;
      }

      base64 = base64.replace(/^data:application\/octet-stream/, 'data:image/jpeg');

      const matches = base64.match(/^data:image\/(\w+);base64,(.+)$/);
      if (!matches) {
         ctx.body = { code: 400, message: 'base64格式不正确' };
         return;
      }

      const ext = matches[1];
      const data = matches[2];
      const buffer = Buffer.from(data, 'base64');

      // 配置七牛 SDK
      const mac = new qiniu.auth.digest.Mac(accessKey, secretKey);
      const options = {
         scope: bucket,
      };
      const putPolicy = new qiniu.rs.PutPolicy(options);
      const uploadToken = putPolicy.uploadToken(mac);
      const config = new qiniu.conf.Config();
      config.zone = qiniu.zone.Zone_z0; // 根据你的存储区域调整
      // 七牛接口:putBase64
      const formUploader = new qiniu.form_up.FormUploader(config);
      const putExtra = new qiniu.form_up.PutExtra();
      let keyPath_ = `ai-query${keyPath ? '/' + keyPath : ''}`;
      const key = `${keyPath_}/${fileName || uuid.v4() + '.' + ext}`;

      const putBase64 = (uploadToken, key, buffer) => {
         return new Promise((resolve, reject) => {
            formUploader.put(uploadToken, key, buffer, putExtra, function (err, body, info) {
               if (err) {
                  reject(err);
               } else {
                  resolve({ body, info });
               }
            });
         });
      };

      const result = await putBase64(uploadToken, key, buffer);

      if (result.info.statusCode === 200) {
         const { models } = ctx.app.fs.dc;
         const { ImageUnderstandingAppKey } = ctx.app.fs.config.fastGpt;
         const imgRes = await models.ImgRecognition.create({
            path: result.body.key,
            addTime: moment().format(),
         })

         superagent
            .post(`${ctx.app.fs.config.fastGpt.apiUrl}/api/v1/chat/completions`)
            .send({
               "stream": false,
               "detail": true,
               "messages": [
                  {
                     "role": "user",
                     "content": [
                        {
                           "type": "image_url",
                           "image_url": {
                              "url": `${domain}/${result.body.key}`
                           }
                        },
                        {
                           "type": "text",
                           // "text": `判断图像内有多少车辆,分别是什么车型;如果有摩托车或电动车,则为异常;将返回结果json化,并且结果可以使用 JSON.parse 进行解析;用 warning 字段标记异常,返回的json格式如下:{"warning":true,"describe":"结果描述"}`
                           "text": prompt || `当前图像内容为摄像头监测画面,请对以下自然灾害及工程结构失效场景进行智能识别、分类与异常判断:
识别目标及标准:
1. 倒树检测:识别由于强风、洪水或老化等原因导致的树木倾倒现象.需准确区分正常直立树木与明显倒伏状态(如树干接近水平,或根部抬起).
2. 桥梁垮塌识别:检测桥梁结构整体或局部坍塌事件,包括但不限于桥面断裂、桥墩台沉降、桥体错位、桥板断裂等典型破坏特征.
3. 落梁事件判断:专项识别桥梁梁体脱离支座发生坠落、倾斜移位或悬挂等异常情况.需检测梁体与支座的连接状态异常.
4. 岩体崩塌监测:监测山体边坡区域岩块剥落、滚石、大规模滑坡等现象,需结合地形坡度、岩层结构特征及崩塌物形态,判断是否存在明显地质灾害征兆.

返回结果要求:
当检测到以上任一异常现象,结果标记 'warning:true',并在 'describe' 字段中简要描述具体异常类型与场景.
当未检测到异常,结果标记 'warning:false',并在 'describe' 字段说明当前状态正常.
返回JSON格式示例:
{"warning":true,"describe":""}
该结果应支持 JSON.parse 解析.`
                        }
                     ]
                  }
               ]
            })
            .set({
               Authorization: `Bearer ${ImageUnderstandingAppKey}`,
               "Content-Type": "application/json",
            }).then((res) => {
               reportFastgptResponse({
                  ctx,
                  applicationId: 'exp-scene',
                  actionId: `scene:${imgRes.id}`,
                  responseBody: res.body,
                  appKey: ImageUnderstandingAppKey,
               });
               const contentMd = res.body.choices[0].message
                  .content?.replaceAll('\n', '')
                  // .replaceAll('```', '')
                  // .replaceAll('json', '')
                  .replaceAll(' ', '');
               const match = contentMd.match(/```json\s*([\s\S]*?)```/)[1];
               const content = JSON.parse(match);
               const { warning, describe } = content
               imgRes.update({
                  recognitionTime: moment().format(),
                  warning,
                  describe,
               })
            }).catch((err) => {
               ctx.logger.log(err);
               imgRes.update({
                  warning: false,
                  describe: '图像理解失败',
               })
            })

         ctx.body = {
            code: 200,
            message: '上传成功',
            url: `${domain}/${result.body.key}`,
         };
      } else {
         ctx.body = { code: 500, message: '上传失败', detail: result.info };
      }
   } catch (error) {
      ctx.body = { code: 500, message: '服务器异常', error };
   }
}


module.exports.vrReslt = async (ctx, next) => {
   try {
      const { models, ORM: { Op } } = ctx.app.fs.dc;
      const { startTime } = ctx.query;
      let findOption = {}
      if (startTime) {
         findOption.addTime = {
            [Op.gte]: startTime,
         }
      }
      const recognitionRes = await models.ImgRecognition.findAll({
         where: findOption,
         order: [['addTime', 'DESC']],
         raw: true
      })

      ctx.body = recognitionRes;
   } catch (error) {
      ctx.body = { code: 400, message: '查询失败', error };
   }
}

/**
 * 功能：上报视频识别模块的播放业务调用。
 * 使用场景：前端点击“播放视频”并成功创建播放器后调用。
 *
 * 入参：deviceSerial 摄像头序列号；hasPlayUrl 是否已获取播放地址。
 * 返回：上报是否成功。
 * 注意：该接口只做业务调用上报，不保存 accessToken。
 */
module.exports.reportPlay = async (ctx, next) => {
   try {
      const { deviceSerial, hasPlayUrl, userId } = ctx.request.body || {};
      const normalizedDeviceSerial = String(deviceSerial || '').trim();

      if (!normalizedDeviceSerial) {
         ctx.status = 400;
         ctx.body = { message: '缺少摄像头序列号' };
         return;
      }

      const eventId = `exp-scene:video-play:${uuid.v4()}`;
      const resolvedUserId = resolveContextUserId(ctx, userId);
      const analyticsConfig = ctx.app.fs.config?.analytics || {};
      const hasAnalyticsConfig = Boolean(
         (analyticsConfig.centerUrl || process.env.AI_CENTER_URL) &&
         (analyticsConfig.keyId || process.env.ANALYTICS_HMAC_KEY_ID) &&
         (analyticsConfig.secret || process.env.ANALYTICS_HMAC_SECRET)
      );
      const reported = await reportBusinessCall({
         ctx,
         applicationId: 'exp-scene',
         eventId,
         userId: resolvedUserId,
         traceId: eventId,
         reportContext: {
            deviceSerial: normalizedDeviceSerial,
            hasPlayUrl: Boolean(hasPlayUrl),
         },
      });

      ctx.body = {
         reported,
         reason: reported
            ? ''
            : (!resolvedUserId ? 'missing_user_id' : (!hasAnalyticsConfig ? 'missing_analytics_config' : 'report_failed')),
      };
   } catch (error) {
      ctx.logger.error('[videoRecognition] 视频播放上报失败', error);
      ctx.status = 400;
      ctx.body = { message: '视频播放上报失败' };
   }
}