运维服务中台
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

741 lines
37 KiB

const moment = require('moment')
module.exports = function (app, opts) {
const alarmsPush = app.fs.scheduleInit(
{
interval: '12 */1 * * * *',
// immediate: true, // dev
proRun: true,
},
async () => {
try {
const { models, ORM: sequelize } = app.fs.dc
const { clickHouse } = app.fs
const { database: anxinyun } = clickHouse.anxinyun.opts.config
const { pushBySms, pushByEmail, sendNoticeToWeb } = app.fs.utils
const curMinOfYear = moment().diff(moment().startOf('year'), 'minutes')
const configListRes = await models.AlarmPushConfig.findAll({
where: {
del: false
},
order: ['id'],
include: [{
model: models.ProjectCorrelation,
where: {
del: false
},
required: true
}],
})
let pepProjectIds = new Set()
for (let { dataValues: c } of configListRes) {
if (c.projectCorrelation.pepProjectId) {
pepProjectIds.add(c.projectCorrelation.pepProjectId)
}
}
const pepProjectRes = pepProjectIds.size ?
await clickHouse.projectManage.query(
`
SELECT
t_pim_project.id AS id,
t_pim_project.project_name AS project_name,
t_pim_project.isdelete AS isdelete,
t_pim_project_construction.construction_status_id AS construction_status_id,
t_pim_project_state.construction_status AS construction_status
FROM t_pim_project
LEFT JOIN t_pim_project_construction
ON t_pim_project.id = t_pim_project_construction.project_id
LEFT JOIN t_pim_project_state
ON t_pim_project_construction.construction_status_id = t_pim_project_state.id
WHERE id IN (${[...pepProjectIds].join(',')})
`
).toPromise() :
[]
for (let { dataValues: c } of configListRes) {
if (c.tacticsParams && c.tactics) {
const { projectCorrelation, strucId, pomsProjectId, } = c
const { interval, deviceProportion } = c.tacticsParams
if (
curMinOfYear % parseInt(interval) == 0
) {
const corPepProject = projectCorrelation.pepProjectId ?
pepProjectRes.find(p => p.id == projectCorrelation.pepProjectId)
: null
if (
!projectCorrelation.pepProjectId
|| (
corPepProject
&& c.timeType.some(ct => ct == corPepProject.construction_status_id)
)
) {
const { anxinProjectId, pepProjectId } = projectCorrelation
// 查当前 poms 下的结构物 并把不包含的去掉
// 可能有结构物已解绑
const strucListRes = strucId.length && anxinProjectId.length ?
await clickHouse.anxinyun.query(
`
SELECT
DISTINCT id,
t_structure.id AS id,
t_structure.name AS name,
t_structure.iota_thing_id AS iotaThingId
FROM
t_project
LEFT JOIN
t_project_structure
ON t_project_structure.project = t_project.id
LEFT JOIN
t_project_structuregroup
ON t_project_structuregroup.project = t_project.id
LEFT JOIN
t_structuregroup_structure
ON t_structuregroup_structure.structuregroup = t_project_structuregroup.structuregroup
LEFT JOIN
t_project_construction
ON t_project_construction.project = t_project.id
LEFT JOIN
t_structure_site
ON t_structure_site.siteid = t_project_construction.construction
RIGHT JOIN
t_structure
ON t_structure.id = t_project_structure.structure
OR t_structure.id = t_structuregroup_structure.structure
OR t_structure.id = t_structure_site.structid
WHERE
project_state != -1
AND t_project.id IN (${anxinProjectId.join(',')})
AND t_structure.id IN (${strucId.join(',')})
`
).toPromise() :
[]
let strucThingId = []
let searchStrucIds = strucListRes.map(s => {
if (s.iotaThingId) {
strucThingId.push(s.iotaThingId)
}
return s.id
})
// 开发测试用的数据
// searchStrucIds = searchStrucIds.concat([991, 1052, 700])
if (searchStrucIds.length) {
searchStrucIds.unshift(-1)
}
let pepProjectName = pepProjectId ?
pepProjectRes.find(p => p.id == pepProjectId).project_name
: projectCorrelation.name
let emailTitle = `${pepProjectName}-${c.name}-`
let emailSubTitle = ''
let dataAlarmOption = []
let dataAlarmGroupOption = []
let dataAlarms = []
let videoAlarmWhereOption = []
let videoAlarms = []
let appAlarmWhereOption = {
confirmTime: null,
}
let appAlarms = []
let deviceCount = 0
let alarmDeviceCount = 0
let cameraCount = 0
let alarmCameraCount = 0
// 判断推送策略
let nowTime = moment().startOf('minute')
let pointTime =
moment()
.subtract(
parseInt(interval),
// + 1440 * 365,
'minute'
)
.startOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
let newAddStartTime = pointTime
let newAddEndTime = nowTime.clone()
if (c.tactics == 'immediately') {
// !查所有未解决告警 所以时间范围大可不必
// dataAlarmOption.push(`StartTime >= '${pointTime}'`);
// appAlarmWhereOption.createTime = { $gte: pointTime }
// videoAlarmWhereOption.push(`camera_status_alarm.create_time >= '${pointTime}'`)
emailTitle += `即时推送服务`
emailSubTitle += `截止${moment(pointTime).format('YYYY年MM月DD日 HH时mm分')}-${moment(nowTime).format('HH时mm分')}`
} else if (c.tactics == 'continue' || c.tactics == 'abnormal_rate') {
// 新增的应该是上一个时间节点到上上个节点之间
dataAlarmOption.push(`StartTime <= '${pointTime}'`);
appAlarmWhereOption.createTime = { $lte: pointTime }
videoAlarmWhereOption.push(`camera_status_alarm.create_time <= '${pointTime}'`)
// 新增的应该是上一个时间节点到上上个节点之间
newAddStartTime = moment(pointTime).subtract(parseInt(interval)).format('YYYY-MM-DD HH:mm:ss')
newAddEndTime = pointTime
if (c.tactics == 'continue') {
emailTitle += `持续时长推送服务`
emailSubTitle += `告警持续时长超${interval}分钟的告警源,详情如下:`
} else {
if (c.alarmType.includes('data_outages') || c.alarmType.includes('data_exception')) {
// 查了设备异常率 去安心云查当前项目下的设备数量
let deviceCountRes =
strucThingId.length ?
await clickHouse.iot.query(`
SELECT count(DeviceId) AS count FROM device WHERE ThingId IN (${strucThingId.map(t => `'${t}'`).join(',')}, '-1')
`).toPromise()
: []
deviceCount = deviceCountRes.length ? deviceCountRes[0].count : 0
}
if (c.alarmType.includes('video_exception')) {
// 查了视频异常 去安心云查 接入的 萤石 设备数量
cameraCount = searchStrucIds.length ?
(await clickHouse.anxinyun.query(`
SELECT count(*) AS count FROM t_video_ipc
WHERE structure IN (${searchStrucIds.join(',')})
`).toPromise())[0].count
: 0
}
emailTitle += `异常率推送服务`
emailSubTitle += `持续产生时间超过${interval}分钟的异常设备数量${interval}个,异常率达到项目或结构物内设备总数量${parseInt(deviceCount) + parseInt(cameraCount)}个的 --%,详情如下`
}
}
emailTitle += '——POMS飞尚运维中台'
// 判断告警数据范围
if (c.alarmType.includes('data_outages')) {
dataAlarmGroupOption.push(1)
}
if (c.alarmType.includes('data_exception')) {
dataAlarmGroupOption.push(2)
}
if (c.alarmType.includes('strategy_hit')) {
dataAlarmGroupOption.push(3)
}
if (c.alarmType.includes('video_exception')) {
videoAlarms = searchStrucIds.length ? await clickHouse.vcmp.query(
`
SELECT
cameraAlarm.cameraId AS cameraId,
cameraAlarm.cameraName AS cameraName,
cameraAlarm.cameraSerialNo AS cameraSerialNo,
cameraAlarm.cameraChannelNo AS cameraChannelNo,
cameraAlarm.alarmId AS alarmId,
cameraAlarm.createTime AS createTime,
cameraAlarm.confirmContent AS confirmContent,
cameraAlarm.confirmTime AS confirmTime,
cameraAlarm.autoRestore AS autoRestore,
camera_status.describe AS statusDescribe,
camera_kind.kind AS cameraKind,
"gbCamera".online AS cameraOnline,
anxinIpc.t_video_ipc.name AS anxinIpcPosition,
anxinStation.id AS anxinStationId,
anxinStation.name AS anxinStationName,
anxinStruc.name AS strucName,
anxinStruc.id AS strucId
FROM (
SELECT
camera.id AS cameraId,
camera.gb_id AS gbId,
camera.name AS cameraName,
camera.kind_id AS cameraKindId,
camera_status_alarm.id AS alarmId,
camera_status_alarm.create_time AS createTime,
camera_status_alarm.status_id AS statusId,
camera_status_alarm.serial_no AS cameraSerialNo,
camera_status_alarm.channel_no AS cameraChannelNo,
camera_status_alarm.confirm AS confirmContent,
camera_status_alarm.auto_restore AS autoRestore,
camera_status_alarm.confirm_time AS confirmTime
FROM camera_status_alarm
INNER JOIN ${anxinyun}.t_video_ipc
ON toString(${anxinyun}.t_video_ipc.channel_no) = camera_status_alarm.channel_no
AND ${anxinyun}.t_video_ipc.serial_no = camera_status_alarm.serial_no
AND ${anxinyun}.t_video_ipc.structure IN (${searchStrucIds.join(',')})
INNER JOIN camera
ON camera.serial_no = camera_status_alarm.serial_no
AND camera.channel_no = camera_status_alarm.channel_no
AND camera.delete = false
AND camera.recycle_time is null
WHERE
camera_status_alarm.confirm_time IS null
${videoAlarmWhereOption.length ? ` AND ${videoAlarmWhereOption.join(' AND ')}` : ''}
) AS cameraAlarm
LEFT JOIN camera_status
ON cameraAlarm.platform = camera_status.platform
AND cameraAlarm.statusId = camera_status.id
LEFT JOIN "gbCamera"
ON "gbCamera".id = cameraAlarm.gbId
LEFT JOIN camera_kind
ON camera_kind.id = cameraAlarm.cameraKindId
LEFT JOIN ${anxinyun}.t_video_ipc AS anxinIpc
ON toString(anxinIpc.channel_no) = cameraAlarm.cameraChannelNo
AND anxinIpc.serial_no = cameraAlarm.cameraSerialNo
LEFT JOIN ${anxinyun}.t_structure AS anxinStruc
ON anxinStruc.id = anxinIpc.structure
AND anxinStruc.id IN (${searchStrucIds.join(',')})
LEFT JOIN ${anxinyun}.t_video_ipc_station AS anxinIpcStation
ON anxinIpcStation.ipc = anxinIpc.id
LEFT JOIN ${anxinyun}.t_sensor AS anxinStation
ON anxinStation.id = anxinIpcStation.station
ORDER BY cameraAlarm.createTime DESC
`
).toPromise() : []
let returnD = []
let positionD = {}
// 每个设备一个告警
for (let a of videoAlarms) {
if (positionD[a.cameraId]) {
let curD = returnD[positionD[a.cameraId].positionReturnD]
if (a.strucId && !curD.struc.some(s => s.id == a.strucId)) {
curD.struc.push({
id: a.strucId,
projectId: a.projectId,
name: a.strucName
})
}
if (a.anxinStationId && !curD.station.some(s => s.id == a.anxinStationId)) {
curD.station.push({
id: a.anxinStationId,
name: a.anxinStationName,
position: a.anxinIpcPosition
})
}
} else {
let d = {
cameraId: a.cameraId,
cameraName: a.cameraName,
camerOnline: a.cameraOnline,
cameraSerialNo: a.cameraSerialNo,
cameraChannelNo: a.cameraChannelNo,
autoRestore: a.autoRestore,
createTime: a.createTime,
updateTime: a.updateTime,
platform: a.platform,
statusDescribe: a.statusDescribe,
alarmId: a.alarmId,
confirmContent: a.confirmContent,
confirmTime: a.confirmTime,
cameraKind: a.cameraKind,
struc: [],
station: []
}
if (a.strucId) {
d.struc.push({
id: a.strucId,
projectId: a.projectId,
name: a.strucName
})
}
if (a.anxinStationId) {
d.station.push({
id: a.anxinStationId,
name: a.anxinStationName,
position: a.anxinIpcPosition
})
}
returnD.push(d)
positionD[a.cameraId] = {
positionReturnD: returnD.length - 1
}
}
}
let p = 1
videoAlarms = returnD
}
if (c.alarmType.includes('app_exception')) {
appAlarms = await models.AppAlarm.findAll({
where: appAlarmWhereOption,
order: [['createTime', 'DESC']],
include: [{
model: models.App,
include: [{
model: models.ProjectApp,
where: {
projectId: pomsProjectId
},
}]
}]
})
}
if (c.alarmType.includes('device_exception')) {
dataAlarmGroupOption.push(4)
dataAlarmGroupOption.push(5)
}
// 查数据告警 三警合一
if (dataAlarmGroupOption.length && searchStrucIds.length) {
dataAlarmGroupOption.push(-1)
dataAlarmOption.push(`AlarmGroup IN (${dataAlarmGroupOption.join(',')})`)
dataAlarms = await clickHouse.dataAlarm.query(`
SELECT * FROM alarms
WHERE
${`State NOT IN (3, 4) AND `}
StructureId IN (${searchStrucIds.join(',')})
${dataAlarmOption.length ? ' AND ' + dataAlarmOption.join(' AND ') : ''}
ORDER BY StartTime DESC
`).toPromise()
console.log(dataAlarms);
}
let dataAlarmTitle = [{
n: '项目',
k: '',
v: pepProjectName
}, {
n: '结构物',
k: '',
f: (d) => {
return (strucListRes.find(s => s.id == d.StructureId) || { name: '' }).name
}
}, {
n: '告警源名称',
k: 'SourceName'
}, {
n: '告警源类型',
k: '',
f: (d) => {
switch (d.SourceTypeId) {
case 0:
return 'DTU'
case 1:
return '传感器'
case 2:
return '测点'
default:
return ''
}
}
}, {
n: '告警信息',
k: 'AlarmContent'
}, {
n: '告警等级(当前)',
k: '',
f: (d) => {
switch (d.CurrentLevel) {
case 1:
return '一级'
case 2:
return '二级'
case 3:
return '三级'
default:
return ''
}
}
}, {
n: '持续时间',
k: '',
f: (d) => {
return d.StartTime ?
'超过' + moment().diff(moment(d.StartTime), 'minutes') + '分钟' : ''
}
},]
let videoAlarmTitle = [{
n: '项目',
k: '',
v: pepProjectName
}, {
n: '结构物',
k: '',
f: (d) => {
return d.struc.map(ds => ds.name).join('、')
}
}, {
n: '告警源名称',
k: 'cameraName'
}, {
n: '告警源类型',
k: 'cameraKind'
}, {
n: '序列号',
k: 'cameraSerialNo'
}, {
n: '通道号',
k: 'cameraChannelNo'
}, {
n: '测点',
k: '',
f: (d) => {
return d.station.map(ds => ds.name).join('、')
}
}, {
n: '位置',
k: '',
f: (d) => {
return d.station.map(ds => ds.position).join('、')
}
}, {
n: '告警信息',
k: 'statusDescribe'
}, {
n: '持续时间',
k: '',
f: (d) => {
return d.createTime ?
'超过' + moment().diff(moment(d.createTime), 'minutes') + '分钟' : ''
}
},]
let appAlarmTitle = [{
n: '项目',
k: '',
v: pepProjectName
}, {
n: '应用名称',
k: '',
f: (d) => {
return d.app ? d.app.name : ''
}
}, {
n: '异常类型',
k: '',
f: (d) => {
if (d.type == 'element') {
return '元素异常'
} else if (d.type == 'apiError') {
return '接口报错'
} else {
return ''
}
}
}, {
n: '异常信息',
k: 'alarmContent'
}, {
n: 'URL地址',
k: 'cameraName',
f: (d) => {
return d.app && d.app.url ? `<a href="${d.app.url}">${d.app.url}</a>` : ''
}
}, {
n: '持续时间',
k: '',
f: (d) => {
return d.createTime ?
'超过' + moment().diff(moment(d.createTime), 'minutes') + '分钟' : ''
}
},]
let ifEmailSend = false
let tableTitlePostfix = ',详情如下:'
function packageTableTitle (titleArr) {
let tableTitle = '<tr>'
for (let t of titleArr) {
tableTitle += `<th style="background-color:#8faadc; color:#fff; text-align:left">${t.n}</th>`
}
tableTitle += '</tr>'
return tableTitle
}
function packageTableData (data, titleArr) {
let tableData = '<tr>'
for (let t of titleArr) {
if (t.v) {
tableData += `<td>${t.v}</td>`
} else if (t.f) {
tableData += `<td>${t.f(data)}</td>`
} else if (t.k) {
tableData += `<td>${data[t.k]}</td>`
} else {
tableData += `<td></td>`
}
}
tableData += '</tr>'
return tableData
}
function packageAlarmData2Table (titlePrefix, alarmData, alarmTitleArr, keyOfStartTime = 'StartTime') {
if (!alarmData.length) {
return ''
}
ifEmailSend = true
let tableTitlePrefix = titlePrefix + '告警源'
let newAddCount = 0
let alarmHtml = '<table border="1" cellspacing="0" style="border-collapse:collapse;border-spacing:0;border-left:1px solid #888;border-top:1px solid #888;">'
let alarmContent = ''
let alarmHtmlTitle = packageTableTitle(alarmTitleArr)
for (let a of alarmData) {
alarmContent += packageTableData(a, alarmTitleArr)
if (a[keyOfStartTime] && moment(a[keyOfStartTime]).isBetween(newAddStartTime, newAddEndTime)) {
newAddCount++
}
}
tableTitlePrefix +=
c.tactics == 'abnormal_rate' ?
`${alarmData.length}` :
`新增${newAddCount}个,未解决累计${alarmData.length}` + tableTitlePostfix
alarmHtml += `<tr><td colspan="${alarmTitleArr.length}" style="background-color:#ffff00">` + tableTitlePrefix + '</td></tr>'
alarmHtml += alarmHtmlTitle
alarmHtml += alarmContent
alarmHtml += '</table><br/>'
return alarmHtml
}
let dataAlarmG1 = [];
let dataAlarmG2 = [];
let dataAlarmG3 = [];
let dataAlarmG45 = [];
let deviceStatistic = new Set()
for (let d of dataAlarms) {
if (d.AlarmGroup == 1) {
dataAlarmG1.push(d)
} else if (d.AlarmGroup == 2) {
dataAlarmG2.push(d)
} else if (d.AlarmGroup == 3) {
dataAlarmG3.push(d)
} else if (d.AlarmGroup == 4 || d.AlarmGroup == 5) {
dataAlarmG45.push(d)
}
deviceStatistic.add(d.SourceId)
}
if (c.tactics == 'abnormal_rate') {
let rate = ((deviceStatistic.size + videoAlarms.length) / (parseInt(deviceCount) + parseInt(cameraCount)));
if (rate < parseFloat(deviceProportion)) {
continue
}
emailSubTitle = emailSubTitle.replace('--%', rate.toFixed(1) + '%')
}
let html = `
<html>
<head>
<meta charset="utf-8">
<style>
table {
border-collapse:collapse;border-spacing:0;border-left:1px solid #888;border-top:1px solid #888;
}
th,td{
border-right:1px solid #888;border-bottom:1px solid #888;padding:5px 15px;
}
th{
font-weight:bold;
}
</style>
</head>
</html>
<h5 style="margin-bottom:12px">${emailSubTitle}</h5>
`
if (c.alarmType.includes('data_outages')) {
html += packageAlarmData2Table(
'数据中断',
dataAlarmG1,
dataAlarmTitle,
)
}
if (c.alarmType.includes('data_exception')) {
html += packageAlarmData2Table(
'数据异常',
dataAlarmG2,
dataAlarmTitle,
)
}
if (c.alarmType.includes('strategy_hit')) {
html += packageAlarmData2Table(
'策略命中',
dataAlarmG3,
dataAlarmTitle,
)
}
if (c.alarmType.includes('video_exception')) {
html += packageAlarmData2Table(
'视频异常',
videoAlarms,
videoAlarmTitle,
'createTime',
)
}
if (c.alarmType.includes('app_exception')) {
html += packageAlarmData2Table(
'应用异常',
appAlarms,
appAlarmTitle,
'createTime',
)
}
if (c.alarmType.includes('device_exception')) {
html += packageAlarmData2Table(
'设备异常',
dataAlarmG45,
dataAlarmTitle,
)
}
if (ifEmailSend) {
// 查接收人的信息
const receiverRes =
c.receiverPepUserId.length ?
await clickHouse.pepEmis.query(`
SELECT id, name, email FROM user WHERE id IN (${c.receiverPepUserId.join(',')},-1)
`).toPromise()
: []
let receiverId = []
const emails = receiverRes.reduce((arr, r) => {
if (r.email) {
arr.push(r.email)
receiverId.push({ id: r.id, name: r.name })
}
return arr
}, [])
if (emails.length) {
await pushByEmail({
email: emails,
title: emailTitle,
text: '',
html: html
})
//存日志 存动态 socket到前端
let dataToSave = {
time: moment().format(),
pushConfigId: c.id,
cfgName: c.name,//策略名称
tactics: c.tactics,
tacticsParams: c.tacticsParams,
projectCorrelationId: pomsProjectId,
toPepUserIds: receiverId.map(r => r.id)
}
let r = await models.EmailSendLog.create(dataToSave, { returning: true })
let dynamic = {
time: r.dataValues.time,
emailSendId: r.dataValues.id,
projectCorrelationId: r.dataValues.projectCorrelationId,
type: 2//通知
}
await models.LatestDynamicList.create(dynamic);
//消息推送到前端
await sendNoticeToWeb(receiverId, dataToSave);
}
}
}
}
}
}
} catch (error) {
console.error(error);
}
}
)
return {
alarmsPush,
}
}