From f1cde7e65cffba183b579a0a67a3449ca3ca780e Mon Sep 17 00:00:00 2001 From: wenlele Date: Fri, 30 Jun 2023 16:44:49 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=B4=A8=E6=A3=80?= =?UTF-8?q?=E6=83=85=E5=86=B5=E5=9B=BE=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/app/lib/controllers/dataQuality/index.js | 26 ++- api/app/lib/models/quality_check_result.js | 62 ++++++ api/app/lib/routes/dataQuality/index.js | 3 + api/app/lib/routes/latestMetadata/index.js | 24 ++- .../0.0.9/03.alter_t_quality_check_result.sql | 14 ++ .../src/sections/dataQuality/actions/index.js | 2 + .../dataQuality/actions/qualityMonitor.js | 18 ++ .../dataQuality/containers/qualityMonitor.js | 191 ++++++++++++++++-- .../dataService/actions/serviceManagement.js | 5 +- .../dataService/components/resourceModal.js | 8 +- .../containers/serviceManagement.js | 6 +- web/client/src/utils/webapi.js | 3 +- 12 files changed, 327 insertions(+), 35 deletions(-) create mode 100644 api/app/lib/models/quality_check_result.js create mode 100644 scripts/0.0.9/03.alter_t_quality_check_result.sql create mode 100644 web/client/src/sections/dataQuality/actions/qualityMonitor.js diff --git a/api/app/lib/controllers/dataQuality/index.js b/api/app/lib/controllers/dataQuality/index.js index 0bd7250..cdd5ee8 100644 --- a/api/app/lib/controllers/dataQuality/index.js +++ b/api/app/lib/controllers/dataQuality/index.js @@ -1,6 +1,6 @@ 'use strict'; const moment = require('moment') - +const { sequelize, Sequelize } = require('sequelize'); function getStandardDocFolders (opts) { return async function (ctx, next) { @@ -404,7 +404,7 @@ function fetchFiles (opts) { try { const models = ctx.fs.dc.models; const { folderId } = ctx.request.body; - + let fileAll = await models.StandardDoc.findAll({}) let folderAll = await models.StandardDocFolder.findAll({}) @@ -433,6 +433,25 @@ function fetchFiles (opts) { } } +function getQualityInspection (opts) { + return async function (ctx, next) { + + const models = ctx.fs.dc.models; + const { } = ctx.query; + try { + + let res = await models.QualityCheckResult.findAll() || [] + + ctx.status = 200; + ctx.body = res || []; + } catch (error) { + ctx.fs.logger.error(`path: ${ctx.path}, error: ${error}`); + ctx.status = 400; + ctx.body = { message: '查询数据质检详情失败' } + } + } +} + module.exports = { getStandardDocFolders, postStandardDocFolders, @@ -443,5 +462,6 @@ module.exports = { delBusinessRules, getRegularBasis, postFolderFile, - fetchFiles + fetchFiles, + getQualityInspection } \ No newline at end of file diff --git a/api/app/lib/models/quality_check_result.js b/api/app/lib/models/quality_check_result.js new file mode 100644 index 0000000..1b451ba --- /dev/null +++ b/api/app/lib/models/quality_check_result.js @@ -0,0 +1,62 @@ +/* eslint-disable*/ + +'use strict'; + +module.exports = dc => { + const DataTypes = dc.ORM; + const sequelize = dc.orm; + const QualityCheckResult = sequelize.define("qualityCheckResult", { + id: { + type: DataTypes.BIGINT, + allowNull: false, + defaultValue: null, + comment: null, + primaryKey: true, + field: "id", + autoIncrement: true, + unique: "t_quality_check_result_id_uindex" + }, + database: { + type: DataTypes.TEXT, + allowNull: true, + defaultValue: null, + comment: null, + primaryKey: false, + field: "database", + autoIncrement: false, + }, + mode: { + type: DataTypes.TEXT, + allowNull: true, + defaultValue: null, + comment: null, + primaryKey: false, + field: "mode", + autoIncrement: false + }, + total: { + type: DataTypes.BIGINT, + allowNull: true, + defaultValue: null, + comment: null, + primaryKey: false, + field: "total", + autoIncrement: false + }, + unexpected: { + type: DataTypes.BIGINT, + allowNull: true, + defaultValue: null, + comment: null, + primaryKey: false, + field: "unexpected", + autoIncrement: false + } + }, { + tableName: "t_quality_check_result", + comment: "", + indexes: [] + }); + dc.models.QualityCheckResult = QualityCheckResult; + return QualityCheckResult; +}; \ No newline at end of file diff --git a/api/app/lib/routes/dataQuality/index.js b/api/app/lib/routes/dataQuality/index.js index 174919e..a7f04b0 100644 --- a/api/app/lib/routes/dataQuality/index.js +++ b/api/app/lib/routes/dataQuality/index.js @@ -34,4 +34,7 @@ module.exports = function (app, router, opts, AuthCode) { app.fs.api.logAttr['POST/fetchFiles'] = { content: '获取文件夹下文件', visible: true }; router.post('/fetchFiles', model.fetchFiles(opts)); + + app.fs.api.logAttr['GET/quality-inspection'] = { content: '查询数据质检详情', visible: true }; + router.get('/quality-inspection', model.getQualityInspection(opts)); }; diff --git a/api/app/lib/routes/latestMetadata/index.js b/api/app/lib/routes/latestMetadata/index.js index 94eac83..0f192f0 100644 --- a/api/app/lib/routes/latestMetadata/index.js +++ b/api/app/lib/routes/latestMetadata/index.js @@ -175,19 +175,14 @@ module.exports = function (app, router, opts) { } }); + + async function release (apps, opts) { const models = apps.fs.dc.models; - const list = await models.RestfulApi.findAll({ - order: [["id", "desc"]], - include: [{ - model: models.ResourceConsumption, - }], - distinct: true - }) || [] + const list = await models.RestfulApi.findAll() || [] list.map(v => { - opts.exclude.push({ p: v.url, o: 'GET' }); router.get(v.url, async (ctx) => { let message = "获取库表元数据列表失败" @@ -199,13 +194,22 @@ module.exports = function (app, router, opts) { throw '' } else { let tokens - v.resourceConsumptions.map(s => { + let findOne = await models.RestfulApi.findOne({ + where: { url: v.url, method: v.method }, + order: [["id", "desc"]], + include: [{ + model: models.ResourceConsumption, + }], + distinct: true + }); + findOne && findOne.resourceConsumptions.map(s => { if (!tokens && s.token) { tokens = s.token } }) + if (tokens && tokens == token) { - if (v.enabled) { + if (findOne.enabled) { const pool = new Pool({ user: ctx.fs.dc.orm.config.username, host: ctx.fs.dc.orm.config.host, diff --git a/scripts/0.0.9/03.alter_t_quality_check_result.sql b/scripts/0.0.9/03.alter_t_quality_check_result.sql new file mode 100644 index 0000000..6524522 --- /dev/null +++ b/scripts/0.0.9/03.alter_t_quality_check_result.sql @@ -0,0 +1,14 @@ +create table t_quality_check_result +( + id bigserial not null + constraint t_quality_check_result_pkey + primary key, + database text, + mode text, + total bigint, + unexpected bigint +); + +create unique index idx_unique_db_mode + on t_quality_check_result (database, mode); + diff --git a/web/client/src/sections/dataQuality/actions/index.js b/web/client/src/sections/dataQuality/actions/index.js index bb9a933..ca8ee95 100644 --- a/web/client/src/sections/dataQuality/actions/index.js +++ b/web/client/src/sections/dataQuality/actions/index.js @@ -3,9 +3,11 @@ import * as example from './example' import * as documentLibrary from './documentLibrary' import * as ruleLibrary from './ruleLibrary' +import * as qualityMonitor from './qualityMonitor' export default { ...example, ...documentLibrary, ...ruleLibrary, + ...qualityMonitor } \ No newline at end of file diff --git a/web/client/src/sections/dataQuality/actions/qualityMonitor.js b/web/client/src/sections/dataQuality/actions/qualityMonitor.js new file mode 100644 index 0000000..9e20c48 --- /dev/null +++ b/web/client/src/sections/dataQuality/actions/qualityMonitor.js @@ -0,0 +1,18 @@ +'use strict'; + +import { basicAction } from '@peace/utils' +import { ApiTable } from '$utils' + + + +export function getQualityInspection (query = {}) { + return dispatch => basicAction({ + type: 'get', + query, + dispatch: dispatch, + actionType: 'GET_QUALITY_INSPECT', + url: `${ApiTable.getQualityInspection}`, + msg: { error: '查询数据质检详情' }, + reducer: { name: '' } + }); +} diff --git a/web/client/src/sections/dataQuality/containers/qualityMonitor.js b/web/client/src/sections/dataQuality/containers/qualityMonitor.js index e58f133..6b1ca9e 100644 --- a/web/client/src/sections/dataQuality/containers/qualityMonitor.js +++ b/web/client/src/sections/dataQuality/containers/qualityMonitor.js @@ -13,15 +13,10 @@ function MyApplication ({ loading, clientHeight, actions, dispatch, user }) { const { dataQuality } = actions const [query, setQuery] = useState({ page: 0, limit: 10 }); - const [proTableList, setProTableList] = useState({ rows: [], count: 0 }); + const [massData, setMassData] = useState(); const [bistribution, setBistribution] = useState({}) useEffect(() => { - resourceData() - }, []) - - - let resourceData = () => { dispatch(dataQuality.getBusinessRules({})).then(res => { if (res.success) { let data = res.payload.data?.rows @@ -34,7 +29,43 @@ function MyApplication ({ loading, clientHeight, actions, dispatch, user }) { } }) - } + dispatch(dataQuality.getQualityInspection({})).then(res => { + if (res.success) { + let data = { + Consistency: [], //一致性 + Accuracy: [], //准确性 + Completeness: [], //完整性 + Validity: [], //有效性 + Timeliness: [], //及时性 + Conformity: [], //规范性 + } + res.payload.data?.map(s => { + data[s.mode].push({ total: Number(s.total), unexpected: Number(s.unexpected) }) + }) + let list = { + Consistency: {}, //一致性 + Accuracy: {}, //准确性 + Completeness: {}, //完整性 + Validity: {}, //有效性 + Timeliness: {}, //及时性 + Conformity: {}, //规范性 + } + for (let key in data) { + let total = 0 + let unexpected = 0 + data[key]?.map(d => { + total += d.total + unexpected += d.unexpected + }) + list[key] = { total: total, unexpected: unexpected } + } + + setMassData(list) + } + }) + }, []) + + // console.log(massData); const option = { @@ -77,16 +108,16 @@ function MyApplication ({ loading, clientHeight, actions, dispatch, user }) { const [count, setCount] = useState(0); function onChartReady (echarts) { - console.log('echarts is ready', echarts); + // console.log('echarts is ready', echarts); } function onChartClick (param, echarts) { - console.log(param, echarts); + // console.log(param, echarts); setCount(count + 1); }; function onChartLegendselectchanged (param, echarts) { - console.log(param, echarts); + // console.log(param, echarts); }; return
@@ -94,11 +125,11 @@ function MyApplication ({ loading, clientHeight, actions, dispatch, user }) {
业务规则情况
-
- 业务规则个数：{bistribution?.count || 0} 个
+
+ 业务规则个数：{bistribution?.count || 0} 个
+
数据质检情况
+
+
diff --git a/web/client/src/sections/dataService/actions/serviceManagement.js b/web/client/src/sections/dataService/actions/serviceManagement.js index 99a19cc..b8012bc 100644 --- a/web/client/src/sections/dataService/actions/serviceManagement.js +++ b/web/client/src/sections/dataService/actions/serviceManagement.js @@ -16,14 +16,15 @@ export function getServiceManagement (query = {}) { } -export function postServiceManagement (data = {}) { +export function postServiceManagement (data = {}, distinguish) { + let option = distinguish ? data?.enabled ? 'REST服务启用' : 'REST服务禁用' : '编辑REST服务' return dispatch => basicAction({ type: 'post', data, dispatch: dispatch, actionType: 'POST_SERVICE_MANAGEMENT', url: `${ApiTable.serviceManagement}`, - msg: { option: '编辑REST服务' }, + msg: { option: option }, reducer: { name: '' } }); } diff --git a/web/client/src/sections/dataService/components/resourceModal.js b/web/client/src/sections/dataService/components/resourceModal.js index 89fbebd..a879275 100644 --- a/web/client/src/sections/dataService/components/resourceModal.js +++ b/web/client/src/sections/dataService/components/resourceModal.js @@ -38,19 +38,21 @@ const ResourceModal = (props) => { handleOk(null)} - onCancel={()=>close()}> + onCancel={() => close()}>
- + - + 删除 -{/* // } */} + {/* // } */} {record?.enabled ? { dispatch(dataService.postServiceManagement({ id: record.id, name: record?.name, enabled: false - })).then(res => { + }, true)).then(res => { if (res.success) { resourceData({ keyword }) } @@ -98,7 +98,7 @@ function ServiceManagement ({ loading, clientHeight, actions, dispatch, }) { : { dispatch(dataService.postServiceManagement({ id: record.id, name: record?.name, enabled: true - })).then(res => { + }, true)).then(res => { if (res.success) { resourceData({ keyword }) } diff --git a/web/client/src/utils/webapi.js b/web/client/src/utils/webapi.js index 95f9d12..0245b23 100644 --- a/web/client/src/utils/webapi.js +++ b/web/client/src/utils/webapi.js @@ -96,6 +96,7 @@ export const ApiTable = { delBusinessRules: 'business-rules/{id}', regularBasis: 'regular-basis', fetchFiles: 'fetchFiles', + getQualityInspection: 'quality-inspection', //数据安全规范上传 specifications: 'data-security/specifications', @@ -113,7 +114,7 @@ export const ApiTable = { //REST服务 serviceManagement: 'service-management', delServiceManagement: 'service-management/{id}', - lookField:'lookField', + lookField: 'lookField', }; From a8480a870716b226e35da72c8823db4fcbcedd9a Mon Sep 17 00:00:00 2001 From: wenlele Date: Fri, 30 Jun 2023 16:52:58 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E7=94=B3=E8=AF=B7rest=E6=9C=8D=E5=8A=A1?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E5=BC=B9=E7=AA=97=E4=B8=AD=E5=90=8D=E7=A7=B0?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/sections/dataService/components/resourceModal.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/web/client/src/sections/dataService/components/resourceModal.js b/web/client/src/sections/dataService/components/resourceModal.js index a879275..37f63a6 100644 --- a/web/client/src/sections/dataService/components/resourceModal.js +++ b/web/client/src/sections/dataService/components/resourceModal.js @@ -41,10 +41,10 @@ const ResourceModal = (props) => { onCancel={() => close()}> + rules={[{ required: true, message: '名称不可空' }]}>