const qiniu = require('qiniu'); const moment = require('moment'); const uuid = require('uuid'); const superagent = require('superagent'); const { reportBusinessCall, reportFastgptResponse, resolveContextUserId } = require('../services/dashboardReporter'); module.exports.vrdemo = async (ctx, next) => { let { url, base64, fileName, key: keyPath, prompt } = ctx.request.body; try { const { qiniu: { dmn: domain, bkt: bucket, ak: accessKey, sk: secretKey } } = ctx.config; if (url) { const response = await superagent .get(url) .responseType('blob') // 或者 'arraybuffer',主要是为了拿到 buffer .buffer(true); // 强制 superagent 处理成 buffer const contentType = response.headers['content-type']; // 获取图片类型 const buffer = response.body; // 图片数据是 buffer base64 = `data:${contentType};base64,` + buffer.toString('base64'); } if (!base64) { ctx.body = { code: 400, message: '缺少base64参数' }; return; } base64 = base64.replace(/^data:application\/octet-stream/, 'data:image/jpeg'); const matches = base64.match(/^data:image\/(\w+);base64,(.+)$/); if (!matches) { ctx.body = { code: 400, message: 'base64格式不正确' }; return; } const ext = matches[1]; const data = matches[2]; const buffer = Buffer.from(data, 'base64'); // 配置七牛 SDK const mac = new qiniu.auth.digest.Mac(accessKey, secretKey); const options = { scope: bucket, }; const putPolicy = new qiniu.rs.PutPolicy(options); const uploadToken = putPolicy.uploadToken(mac); const config = new qiniu.conf.Config(); config.zone = qiniu.zone.Zone_z0; // 根据你的存储区域调整 // 七牛接口:putBase64 const formUploader = new qiniu.form_up.FormUploader(config); const putExtra = new qiniu.form_up.PutExtra(); let keyPath_ = `ai-query${keyPath ? '/' + keyPath : ''}`; const key = `${keyPath_}/${fileName || uuid.v4() + '.' + ext}`; const putBase64 = (uploadToken, key, buffer) => { return new Promise((resolve, reject) => { formUploader.put(uploadToken, key, buffer, putExtra, function (err, body, info) { if (err) { reject(err); } else { resolve({ body, info }); } }); }); }; const result = await putBase64(uploadToken, key, buffer); if (result.info.statusCode === 200) { const { models } = ctx.app.fs.dc; const { ImageUnderstandingAppKey } = ctx.app.fs.config.fastGpt; const imgRes = await models.ImgRecognition.create({ path: result.body.key, addTime: moment().format(), }) superagent .post(`${ctx.app.fs.config.fastGpt.apiUrl}/api/v1/chat/completions`) .send({ "stream": false, "detail": true, "messages": [ { "role": "user", "content": [ { "type": "image_url", "image_url": { "url": `${domain}/${result.body.key}` } }, { "type": "text", // "text": `判断图像内有多少车辆,分别是什么车型;如果有摩托车或电动车,则为异常;将返回结果json化,并且结果可以使用 JSON.parse 进行解析;用 warning 字段标记异常,返回的json格式如下:{"warning":true,"describe":"结果描述"}` "text": prompt || `当前图像内容为摄像头监测画面,请对以下自然灾害及工程结构失效场景进行智能识别、分类与异常判断: 识别目标及标准: 1. 倒树检测:识别由于强风、洪水或老化等原因导致的树木倾倒现象.需准确区分正常直立树木与明显倒伏状态(如树干接近水平,或根部抬起). 2. 桥梁垮塌识别:检测桥梁结构整体或局部坍塌事件,包括但不限于桥面断裂、桥墩台沉降、桥体错位、桥板断裂等典型破坏特征. 3. 落梁事件判断:专项识别桥梁梁体脱离支座发生坠落、倾斜移位或悬挂等异常情况.需检测梁体与支座的连接状态异常. 4. 岩体崩塌监测:监测山体边坡区域岩块剥落、滚石、大规模滑坡等现象,需结合地形坡度、岩层结构特征及崩塌物形态,判断是否存在明显地质灾害征兆. 返回结果要求: 当检测到以上任一异常现象,结果标记 'warning:true',并在 'describe' 字段中简要描述具体异常类型与场景. 当未检测到异常,结果标记 'warning:false',并在 'describe' 字段说明当前状态正常. 返回JSON格式示例: {"warning":true,"describe":""} 该结果应支持 JSON.parse 解析.` } ] } ] }) .set({ Authorization: `Bearer ${ImageUnderstandingAppKey}`, "Content-Type": "application/json", }).then((res) => { reportFastgptResponse({ ctx, applicationId: 'exp-scene', actionId: `scene:${imgRes.id}`, responseBody: res.body, appKey: ImageUnderstandingAppKey, }); const contentMd = res.body.choices[0].message .content?.replaceAll('\n', '') // .replaceAll('```', '') // .replaceAll('json', '') .replaceAll(' ', ''); const match = contentMd.match(/```json\s*([\s\S]*?)```/)[1]; const content = JSON.parse(match); const { warning, describe } = content imgRes.update({ recognitionTime: moment().format(), warning, describe, }) }).catch((err) => { ctx.logger.log(err); imgRes.update({ warning: false, describe: '图像理解失败', }) }) ctx.body = { code: 200, message: '上传成功', url: `${domain}/${result.body.key}`, }; } else { ctx.body = { code: 500, message: '上传失败', detail: result.info }; } } catch (error) { ctx.body = { code: 500, message: '服务器异常', error }; } } module.exports.vrReslt = async (ctx, next) => { try { const { models, ORM: { Op } } = ctx.app.fs.dc; const { startTime } = ctx.query; let findOption = {} if (startTime) { findOption.addTime = { [Op.gte]: startTime, } } const recognitionRes = await models.ImgRecognition.findAll({ where: findOption, order: [['addTime', 'DESC']], raw: true }) ctx.body = recognitionRes; } catch (error) { ctx.body = { code: 400, message: '查询失败', error }; } } /** * 功能:上报视频识别模块的播放业务调用。 * 使用场景:前端点击“播放视频”并成功创建播放器后调用。 * * 入参:deviceSerial 摄像头序列号;hasPlayUrl 是否已获取播放地址。 * 返回:上报是否成功。 * 注意:该接口只做业务调用上报,不保存 accessToken。 */ module.exports.reportPlay = async (ctx, next) => { try { const { deviceSerial, hasPlayUrl, userId } = ctx.request.body || {}; const normalizedDeviceSerial = String(deviceSerial || '').trim(); if (!normalizedDeviceSerial) { ctx.status = 400; ctx.body = { message: '缺少摄像头序列号' }; return; } const eventId = `exp-scene:video-play:${uuid.v4()}`; const resolvedUserId = resolveContextUserId(ctx, userId); const analyticsConfig = ctx.app.fs.config?.analytics || {}; const hasAnalyticsConfig = Boolean( (analyticsConfig.centerUrl || process.env.AI_CENTER_URL) && (analyticsConfig.keyId || process.env.ANALYTICS_HMAC_KEY_ID) && (analyticsConfig.secret || process.env.ANALYTICS_HMAC_SECRET) ); const reported = await reportBusinessCall({ ctx, applicationId: 'exp-scene', eventId, userId: resolvedUserId, traceId: eventId, reportContext: { deviceSerial: normalizedDeviceSerial, hasPlayUrl: Boolean(hasPlayUrl), }, }); ctx.body = { reported, reason: reported ? '' : (!resolvedUserId ? 'missing_user_id' : (!hasAnalyticsConfig ? 'missing_analytics_config' : 'report_failed')), }; } catch (error) { ctx.logger.error('[videoRecognition] 视频播放上报失败', error); ctx.status = 400; ctx.body = { message: '视频播放上报失败' }; } }