diff --git a/api/app/lib/controllers/metadataAcquisition/dataSource.js b/api/app/lib/controllers/metadataAcquisition/dataSource.js index bc740af..bd56972 100644 --- a/api/app/lib/controllers/metadataAcquisition/dataSource.js +++ b/api/app/lib/controllers/metadataAcquisition/dataSource.js @@ -6,11 +6,15 @@ function addDataSource(opts) { const models = ctx.fs.dc.models; try { - const { name } = ctx.request.body - const checkName = await models.DataSource.findOne({ where: { name, name } }); + const { name, mountPath } = ctx.request.body + const checkName = await models.DataSource.findOne({ where: { name: name } }); + const checkMountPath = await models.DataSource.findOne({ where: { mountPath: mountPath } }); if (checkName) { ctx.status = 400; ctx.body = { message: '该数据源名称已存在' } + } else if (checkMountPath) { + ctx.status = 400; + ctx.body = { message: '该资源路径已被使用' } } else { let rslt = ctx.request.body; await models.DataSource.create(Object.assign({}, rslt)) diff --git a/api/app/lib/controllers/metadataAcquisition/taskHandle.js b/api/app/lib/controllers/metadataAcquisition/taskHandle.js index b69b509..89810c8 100644 --- a/api/app/lib/controllers/metadataAcquisition/taskHandle.js +++ b/api/app/lib/controllers/metadataAcquisition/taskHandle.js @@ -46,8 +46,9 @@ async function handleTask(app, task) { }//目录树下库只会存在一个 判断是否有库类型元数据 没有则新增库和表元数据 有库数据则比较更新表类型元数据 全量更新字段索引数据 }) let databaseId = databaseFind ? databaseFind.id : null; + + const newTableNames = [] if (databaseFind) { //更新表类型元数据 - const newTableNames = [] Object.keys(tables).forEach(key => { newTableNames.push(key) }); //删除不存在表元数据 @@ -108,45 +109,74 @@ async function handleTask(app, task) { } }) - //字段/索引/外键全量更新 先删除之前的字段 再录入新的数据 - await models.MetadataDatabase.destroy({ + //字段索引外键上一次存储集合 + const tableChildrens = await models.MetadataDatabase.findAll({ where: { type: { $in: ['字段', '索引', '外键'] }, catalog: dataSource.mountPath, } }) - - const fieldBodys = [] - for (let table of metaDatabaseTables) { - Object.keys(tables[table.name].structures).forEach(key => { - dataToSave.parent = table.id; - dataToSave.name = tables[table.name].structures[key].comment || key; - dataToSave.code = key; - dataToSave.type = '字段'; - const tableObj = { ...dataToSave } - fieldBodys.push(tableObj) + let fieldBodys = [] + let fieldRslt = null; + if (tableChildrens.length == 0) { //首次执行定时任务新增 + for (let table of metaDatabaseTables) { + const children = handleAddTableChildren(dataToSave, tables, table); + fieldBodys = fieldBodys.concat(children) + } + fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys); + } else {//更新数据 + const deleteIds = [];//字段索引外键删除id集合 + const deleteParentIds = [];//删除表集合 + tableChildrens.forEach(s => { + let table = metaDatabaseTables.find(x => x.id == s.parent) + if (table) { + if (s.type == '字段' && !tables[table.name].structures[s.code]) { + deleteIds.push(s.id) + } + if (s.type == '外键' && !tables[table.name].foreignKeys.find(x => x.columnName == s.code)) { + deleteIds.push(s.id) + } + if (s.type == '索引' && !tables[table.name].indexes.find(x => x.name == s.code)) { + deleteIds.push(s.id) + } + } else { + if (!deleteParentIds.find(n => n == s.parent)) deleteParentIds.push(s.parent) + } }) - tables[table.name].foreignKeys.forEach(v => { - dataToSave.parent = table.id; - dataToSave.name = v.columnName; - dataToSave.code = v.columnName; - dataToSave.type = '外键'; - const tableObj = { ...dataToSave } - fieldBodys.push(tableObj) + //判断新增- 新增表 新增表字段 + Object.keys(tables).forEach(key => { + const table = metaDatabaseTables.find(s => s.code == key) + if (tableChildrens.find(s => s.parent == table.id)) { + const children = handleAddTableChildren(dataToSave, tables, table, tableChildrens); + fieldBodys = fieldBodys.concat(children) + } else { + const children = handleAddTableChildren(dataToSave, tables, table); + fieldBodys = fieldBodys.concat(children) + } }) - tables[table.name].indexes.forEach(v => { - dataToSave.parent = table.id; - dataToSave.name = v.name; - dataToSave.code = v.name; - dataToSave.type = '索引'; - const tableObj = { ...dataToSave } - fieldBodys.push(tableObj) + //删除不存在的字段索引 外键 + await models.MetadataDatabase.destroy({ + where: { + type: { $in: ['字段', '索引', '外键'] }, + catalog: dataSource.mountPath, + $or: [ + { parent: { $in: deleteParentIds } }, + { id: { $in: deleteIds } } + ] + } }) + + + //增加新增的字段索引外键 + if (fieldBodys.length > 0) { + fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys); + } else { + fieldRslt = true; + } } - const fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys); if (fieldRslt) { const endTime = moment() const logBody = { @@ -168,10 +198,18 @@ async function handleTask(app, task) { } catch (error) { // await transaction.rollback(); const endTime = moment() + let message = error.message ? error.message : error + //空表auto-sequelize会抛出异常 提示信息处理 + if (error.message && error.message.indexOf('No description found for') > -1) { + if (error.message.split('.') && error.message.split('.').length > 0) { + message = error.message.split('.')[0].split('"')[1] + '未定义任何字段' + } + } + const logBody = { task: task.id, success: false, - details: '采集失败' + JSON.stringify(error).substring(0, 248), + details: '采集失败:' + JSON.stringify(message).substring(0, 247), startTime: startTime, endTime: endTime } @@ -181,12 +219,52 @@ async function handleTask(app, task) { if (task.retried && task.retryCount && task.retryTime && taskRetryIndex[task.id] < task.retryCount) { setTimeout(() => { handleTask(app, task) - }, 1000 * 60 * task.retryCount); + }, 1000 * 60 * task.retryTime); } app.fs.logger.error(`sechedule: handleTask, error: ${error}`); } } +//处理字段 索引 外键 新增body +function handleAddTableChildren(dataToSave, tables, table, tableChildrens) { + const fieldBodys = []; + Object.keys(tables[table.name].structures).forEach(key => { + if (!tableChildrens || !tableChildrens.find(s => s.code == key)) { + dataToSave.parent = table.id; + dataToSave.name = tables[table.name].structures[key].comment || key; + dataToSave.code = key; + dataToSave.type = '字段'; + const tableObj = { ...dataToSave } + fieldBodys.push(tableObj) + } + }) + + tables[table.name].foreignKeys.forEach(v => { + if (!tableChildrens || !tableChildrens.find(s => s.code == v.columnName)) { + dataToSave.parent = table.id; + dataToSave.name = v.columnName; + dataToSave.code = v.columnName; + dataToSave.type = '外键'; + const tableObj = { ...dataToSave } + fieldBodys.push(tableObj) + } + + }) + + tables[table.name].indexes.forEach(v => { + if (!tableChildrens || !tableChildrens.find(s => s.code == v.name)) { + dataToSave.parent = table.id; + dataToSave.name = v.name; + dataToSave.code = v.name; + dataToSave.type = '索引'; + const tableObj = { ...dataToSave } + fieldBodys.push(tableObj) + } + }) + + return fieldBodys; +} + function createDbOptions(params) { const dbOptions = { database: params.database, diff --git a/web/client/src/layout/components/sider/index.js b/web/client/src/layout/components/sider/index.js index 3c1e6b2..e0ccbb0 100644 --- a/web/client/src/layout/components/sider/index.js +++ b/web/client/src/layout/components/sider/index.js @@ -3,6 +3,7 @@ import { Menu } from 'antd'; const JumpUrls = [ { url: '/risk/hiddenrectification_approval', keys: 'riskHiddenrectification_approval' }, { url: '/safetymanage/hiddenrectification_approval', keys: 'hiddenrectification_approval' }, + { url: '/metadataManagement/latestMetadata', keys: 'latestMetadata' }, ] const Sider = (props) => { const [items, setItems] = useState([]) diff --git a/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js b/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js index a3fb950..2f114d6 100644 --- a/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js +++ b/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js @@ -1,5 +1,5 @@ import React, { useEffect, useState } from 'react' -import { Button, Form, Input, Row, Col } from 'antd'; +import { Button, Spin, Input, Row, Col } from 'antd'; import { ProForm, ProFormSelect, @@ -7,10 +7,11 @@ import { ProFormText, ProFormTreeSelect } from '@ant-design/pro-form'; +import { push } from 'react-router-redux'; import '../../style.less'; function StepOne(props) { - const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData, dataSources } = props; + const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData, dataSources, dispatch, refresh, loading, editData } = props; const formRef = React.createRef(); const initialValues = stepOneValues ? stepOneValues : { adapterName: 'PostgreSQL采集适配器', @@ -19,7 +20,7 @@ function StepOne(props) { // mountPath: 1, } - const formItemLayout = { labelCol: { span: 3 }, wrapperCol: { span: 10 } }; + const formItemLayout = { labelCol: { span: 3 }, wrapperCol: { span: 12 } }; const getTreeNodeData = (dataSource, parent, key) => { let treeData = []; @@ -40,8 +41,8 @@ function StepOne(props) { } return treeData } - const treeDataFilter = treeData ? getTreeNodeData(treeData, null, 'rc') : [] - return <> + const treeDataFilter = treeData && dataSources?.rows?.length > 0 ? getTreeNodeData(treeData, null, 'rc') : [] + return */} - 0 ? { return treeDataFilter || []; @@ -137,7 +139,23 @@ function StepOne(props) { label: 'title', }, }} - /> + addonAfter={!editData && <> + { + dispatch(push(`/metadataManagement/latestMetadata`)); + }} style={{ marginRight: 8 }}>新建 + { + refresh() + }}>刷新 + } + disabled={editData} + /> : } - + } export default StepOne \ No newline at end of file diff --git a/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepThree.js b/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepThree.js index 7bc86e7..d1bb9ad 100644 --- a/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepThree.js +++ b/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepThree.js @@ -114,7 +114,7 @@ function StepThree(props) { { const query = { - limit: search ? 10 : pageSize || 10, - page: search ? 1 : currentPage || 1, + // limit: search ? 10 : pageSize || 10, + // page: search ? 1 : currentPage || 1, name: searchValue } dispatch(actions.metadataAcquisition.getDataSources(query)); } - const { data: treeData = [] } = useFsRequest({ url: ApiTable.getResourceCatalog }); + const { data: treeData = [] } = useFsRequest({ + url: ApiTable.getResourceCatalog, + refreshDeps: [refreshTree] + }); useEffect(() => { dispatch(actions.metadataAcquisition.getAdapters()) @@ -141,6 +145,10 @@ function DataSourceManagement(props) { } } } + const refresh = () => { + queryData(); + setRefreshTree(refreshTree + 1) + } return @@ -168,8 +176,9 @@ function DataSourceManagement(props) { total: dataSources?.count, showSizeChanger: true, // showQuickJumper: true, - current: currentPage, - pageSize: pageSize || 10, + // current: currentPage, + // pageSize: pageSize || 10, + defaultPageSize: 10, pageSizeOptions: [10, 20, 50], showTotal: (total) => { return {`共${Math.ceil(total / pageSize)}页，${total}项`} @@ -201,6 +210,7 @@ function DataSourceManagement(props) { visible={visible} onFinish={onFinish} treeData={treeData} + refresh={refresh} {...props} /> }