(*)采集任务 增加空表判断 字段索引外键改为增量更新 数据源配置选择树结构

api/app/lib/controllers/metadataAcquisition/dataSource.js

@@ -6,11 +6,15 @@ function addDataSource(opts) {
 
         const models = ctx.fs.dc.models;
         try {
-            const { name } = ctx.request.body
-            const checkName = await models.DataSource.findOne({ where: { name, name } });
+            const { name, mountPath } = ctx.request.body
+            const checkName = await models.DataSource.findOne({ where: { name: name } });
+            const checkMountPath = await models.DataSource.findOne({ where: { mountPath: mountPath } });
             if (checkName) {
                 ctx.status = 400;
                 ctx.body = { message: '该数据源名称已存在' }
+            } else if (checkMountPath) {
+                ctx.status = 400;
+                ctx.body = { message: '该资源路径已被使用' }
             } else {
                 let rslt = ctx.request.body;
                 await models.DataSource.create(Object.assign({}, rslt))

api/app/lib/controllers/metadataAcquisition/taskHandle.js

@@ -46,8 +46,9 @@ async function handleTask(app, task) {
                 }//目录树下库只会存在一个 判断是否有库类型元数据 没有则新增库和表元数据 有库数据则比较更新表类型元数据 全量更新字段索引数据
             })
             let databaseId = databaseFind ? databaseFind.id : null;
+
+            const newTableNames = []
             if (databaseFind) { //更新表类型元数据
-                const newTableNames = []
                 Object.keys(tables).forEach(key => { newTableNames.push(key) });
 
                 //删除不存在表元数据
@@ -108,45 +109,74 @@ async function handleTask(app, task) {
                     }
                 })
 
-                //字段/索引/外键全量更新 先删除之前的字段 再录入新的数据
-                await models.MetadataDatabase.destroy({
+                //字段索引外键上一次存储集合
+                const tableChildrens = await models.MetadataDatabase.findAll({
                     where: {
                         type: { $in: ['字段', '索引', '外键'] },
                         catalog: dataSource.mountPath,
                     }
                 })
-
-                const fieldBodys = []
-                for (let table of metaDatabaseTables) {
-                    Object.keys(tables[table.name].structures).forEach(key => {
-                        dataToSave.parent = table.id;
-                        dataToSave.name = tables[table.name].structures[key].comment || key;
-                        dataToSave.code = key;
-                        dataToSave.type = '字段';
-                        const tableObj = { ...dataToSave }
-                        fieldBodys.push(tableObj)
+                let fieldBodys = []
+                let fieldRslt = null;
+                if (tableChildrens.length == 0) { //首次执行定时任务新增
+                    for (let table of metaDatabaseTables) {
+                        const children = handleAddTableChildren(dataToSave, tables, table);
+                        fieldBodys = fieldBodys.concat(children)
+                    }
+                    fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys);
+                } else {//更新数据
+                    const deleteIds = [];//字段索引外键删除id集合
+                    const deleteParentIds = [];//删除表集合
+                    tableChildrens.forEach(s => {
+                        let table = metaDatabaseTables.find(x => x.id == s.parent)
+                        if (table) {
+                            if (s.type == '字段' && !tables[table.name].structures[s.code]) {
+                                deleteIds.push(s.id)
+                            }
+                            if (s.type == '外键' && !tables[table.name].foreignKeys.find(x => x.columnName == s.code)) {
+                                deleteIds.push(s.id)
+                            }
+                            if (s.type == '索引' && !tables[table.name].indexes.find(x => x.name == s.code)) {
+                                deleteIds.push(s.id)
+                            }
+                        } else {
+                            if (!deleteParentIds.find(n => n == s.parent)) deleteParentIds.push(s.parent)
+                        }
                     })
 
-                    tables[table.name].foreignKeys.forEach(v => {
-                        dataToSave.parent = table.id;
-                        dataToSave.name = v.columnName;
-                        dataToSave.code = v.columnName;
-                        dataToSave.type = '外键';
-                        const tableObj = { ...dataToSave }
-                        fieldBodys.push(tableObj)
+                    //判断新增-  新增表 新增表字段
+                    Object.keys(tables).forEach(key => {
+                        const table = metaDatabaseTables.find(s => s.code == key)
+                        if (tableChildrens.find(s => s.parent == table.id)) {
+                            const children = handleAddTableChildren(dataToSave, tables, table, tableChildrens);
+                            fieldBodys = fieldBodys.concat(children)
+                        } else {
+                            const children = handleAddTableChildren(dataToSave, tables, table);
+                            fieldBodys = fieldBodys.concat(children)
+                        }
                     })
 
-                    tables[table.name].indexes.forEach(v => {
-                        dataToSave.parent = table.id;
-                        dataToSave.name = v.name;
-                        dataToSave.code = v.name;
-                        dataToSave.type = '索引';
-                        const tableObj = { ...dataToSave }
-                        fieldBodys.push(tableObj)
+                    //删除不存在的字段索引 外键
+                    await models.MetadataDatabase.destroy({
+                        where: {
+                            type: { $in: ['字段', '索引', '外键'] },
+                            catalog: dataSource.mountPath,
+                            $or: [
+                                { parent: { $in: deleteParentIds } },
+                                { id: { $in: deleteIds } }
+                            ]
+                        }
                     })
+
+
+                    //增加新增的字段索引外键
+                    if (fieldBodys.length > 0) {
+                        fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys);
+                    } else {
+                        fieldRslt = true;
+                    }
                 }
 
-                const fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys);
                 if (fieldRslt) {
                     const endTime = moment()
                     const logBody = {
@@ -168,10 +198,18 @@ async function handleTask(app, task) {
     } catch (error) {
         // await transaction.rollback();
         const endTime = moment()
+        let message = error.message ? error.message : error
+        //空表auto-sequelize会抛出异常 提示信息处理
+        if (error.message && error.message.indexOf('No description found for') > -1) {
+            if (error.message.split('.') && error.message.split('.').length > 0) {
+                message = error.message.split('.')[0].split('"')[1] + '未定义任何字段'
+            }
+        }
+        
         const logBody = {
             task: task.id,
             success: false,
-            details: '采集失败' + JSON.stringify(error).substring(0, 248),
+            details: '采集失败:' + JSON.stringify(message).substring(0, 247),
             startTime: startTime,
             endTime: endTime
         }
@@ -181,12 +219,52 @@ async function handleTask(app, task) {
         if (task.retried && task.retryCount && task.retryTime && taskRetryIndex[task.id] < task.retryCount) {
             setTimeout(() => {
                 handleTask(app, task)
-            }, 1000 * 60 * task.retryCount);
+            }, 1000 * 60 * task.retryTime);
         }
         app.fs.logger.error(`sechedule: handleTask, error: ${error}`);
     }
 }
 
+//处理字段 索引 外键 新增body
+function handleAddTableChildren(dataToSave, tables, table, tableChildrens) {
+    const fieldBodys = [];
+    Object.keys(tables[table.name].structures).forEach(key => {
+        if (!tableChildrens || !tableChildrens.find(s => s.code == key)) {
+            dataToSave.parent = table.id;
+            dataToSave.name = tables[table.name].structures[key].comment || key;
+            dataToSave.code = key;
+            dataToSave.type = '字段';
+            const tableObj = { ...dataToSave }
+            fieldBodys.push(tableObj)
+        }
+    })
+
+    tables[table.name].foreignKeys.forEach(v => {
+        if (!tableChildrens || !tableChildrens.find(s => s.code == v.columnName)) {
+            dataToSave.parent = table.id;
+            dataToSave.name = v.columnName;
+            dataToSave.code = v.columnName;
+            dataToSave.type = '外键';
+            const tableObj = { ...dataToSave }
+            fieldBodys.push(tableObj)
+        }
+
+    })
+
+    tables[table.name].indexes.forEach(v => {
+        if (!tableChildrens || !tableChildrens.find(s => s.code == v.name)) {
+            dataToSave.parent = table.id;
+            dataToSave.name = v.name;
+            dataToSave.code = v.name;
+            dataToSave.type = '索引';
+            const tableObj = { ...dataToSave }
+            fieldBodys.push(tableObj)
+        }
+    })
+
+    return fieldBodys;
+}
+
 function createDbOptions(params) {
     const dbOptions = {
         database: params.database,

web/client/src/layout/components/sider/index.js

@@ -3,6 +3,7 @@ import { Menu } from 'antd';
 const JumpUrls = [
     { url: '/risk/hiddenrectification_approval', keys: 'riskHiddenrectification_approval' },
     { url: '/safetymanage/hiddenrectification_approval', keys: 'hiddenrectification_approval' },
+    { url: '/metadataManagement/latestMetadata', keys: 'latestMetadata' },
 ]
 const Sider = (props) => {
     const [items, setItems] = useState([])

web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js

@@ -1,5 +1,5 @@
 import React, { useEffect, useState } from 'react'
-import { Button, Form, Input, Row, Col } from 'antd';
+import { Button, Spin, Input, Row, Col } from 'antd';
 import {
     ProForm,
     ProFormSelect,
@@ -7,10 +7,11 @@ import {
     ProFormText,
     ProFormTreeSelect
 } from '@ant-design/pro-form';
+import { push } from 'react-router-redux';
 
 import '../../style.less';
 function StepOne(props) {
-    const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData, dataSources } = props;
+    const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData, dataSources, dispatch, refresh, loading, editData } = props;
     const formRef = React.createRef();
     const initialValues = stepOneValues ? stepOneValues : {
         adapterName: 'PostgreSQL采集适配器',
@@ -19,7 +20,7 @@ function StepOne(props) {
         // mountPath: 1,
     }
 
-    const formItemLayout = { labelCol: { span: 3 }, wrapperCol: { span: 10 } };
+    const formItemLayout = { labelCol: { span: 3 }, wrapperCol: { span: 12 } };
 
     const getTreeNodeData = (dataSource, parent, key) => {
         let treeData = [];
@@ -40,8 +41,8 @@ function StepOne(props) {
         }
         return treeData
     }
-    const treeDataFilter = treeData ? getTreeNodeData(treeData, null, 'rc') : []
-    return <>
+    const treeDataFilter = treeData && dataSources?.rows?.length > 0 ? getTreeNodeData(treeData, null, 'rc') : []
+    return <Spin spinning={loading || treeDataFilter.length == 0}>
         <ProForm
             title={''}
             initialValues={initialValues}
@@ -112,13 +113,14 @@ function StepOne(props) {
                 label="数据源挂载路径"
             // disabled={true}
             /> */}
-            <ProFormTreeSelect
+            {treeDataFilter.length > 0 ? <ProFormTreeSelect
+                // width={'md'}
                 name="catalogKey"
                 label="数据源挂载路径"
                 placeholder="请选择数据源挂载路径"
                 rules={[{ required: true, message: '请选择数据源挂载路径' }]}
                 allowClear
-                // width={330}
+                width={480}
                 secondary
                 request={async () => {
                     return treeDataFilter || [];
@@ -137,7 +139,23 @@ function StepOne(props) {
                         label: 'title',
                     },
                 }}
-            />
+                addonAfter={!editData && <>
+                    <a onClick={() => {
+                        dispatch(push(`/metadataManagement/latestMetadata`));
+                    }} style={{ marginRight: 8 }}>新建</a>
+                    <a onClick={() => {
+                        refresh()
+                    }}>刷新</a>
+                </>}
+                disabled={editData}
+            /> : <ProFormSelect
+                rules={[{ required: true, message: '请选择' }]}
+                options={[]}
+                name="catalogKey"
+                label="数据源挂载路径"
+                placeholder="请选择数据源挂载路径"
+                disabled={editData}
+            />}
 
             <ProFormTextArea
                 name="description"
@@ -151,7 +169,7 @@ function StepOne(props) {
                 </Button>
             </div>
         </ProForm>
-    </>
+    </Spin>
 }
 
 export default StepOne

web/client/src/sections/metadataAcquisition/components/steps/postgre/stepThree.js

@@ -114,7 +114,7 @@ function StepThree(props) {
                             <ProFormDigit
                                 width={'md'}
                                 rules={[
-                                    // { required: true, message: '请输入重试次数' },
+                                    { required: true, message: '请输入重试次数' },
                                     // { max: 10, message: '重试次数不能大于10个字符' },
                                 ]}
                                 name="retryCount"
@@ -126,7 +126,7 @@ function StepThree(props) {
                             <ProFormDigit
                                 width={'md'}
                                 rules={[
-                                    // { required: true, message: '请输入时间间隔' },
+                                    { required: true, message: '请输入时间间隔' },
                                     // { max: 255, message: '时间间隔长度不能大于255个字符' },
                                 ]}
                                 name="retryTime"

web/client/src/sections/metadataAcquisition/containers/dataSourceManagement.js

@@ -13,16 +13,20 @@ function DataSourceManagement(props) {
     const [searchValue, setSearchValue] = useState('')
     const [visible, setVisible] = useState(false);//是否展示新增编辑模态框
     const [editData, setEditData] = useState(null);//模态框编辑数据
+    const [refreshTree, setRefreshTree] = useState(1);
     const queryData = (search) => {
         const query = {
-            limit: search ? 10 : pageSize || 10,
-            page: search ? 1 : currentPage || 1,
+            // limit: search ? 10 : pageSize || 10,
+            // page: search ? 1 : currentPage || 1,
             name: searchValue
         }
 
         dispatch(actions.metadataAcquisition.getDataSources(query));
     }
-    const { data: treeData = [] } = useFsRequest({ url: ApiTable.getResourceCatalog });
+    const { data: treeData = [] } = useFsRequest({
+        url: ApiTable.getResourceCatalog,
+        refreshDeps: [refreshTree]
+    });
 
     useEffect(() => {
         dispatch(actions.metadataAcquisition.getAdapters())
@@ -141,6 +145,10 @@ function DataSourceManagement(props) {
             }
         }
     }
+    const refresh = () => {
+        queryData();
+        setRefreshTree(refreshTree + 1)
+    }
 
     return <Spin spinning={loading}>
         <Row className='protable-title'>
@@ -168,8 +176,9 @@ function DataSourceManagement(props) {
                 total: dataSources?.count,
                 showSizeChanger: true,
                 // showQuickJumper: true,
-                current: currentPage,
-                pageSize: pageSize || 10,
+                // current: currentPage,
+                // pageSize: pageSize || 10,
+                defaultPageSize: 10,
                 pageSizeOptions: [10, 20, 50],
                 showTotal: (total) => {
                     return <span style={{ fontSize: 15 }}>{`共${Math.ceil(total / pageSize)}页，${total}项`}</span>
@@ -201,6 +210,7 @@ function DataSourceManagement(props) {
                 visible={visible}
                 onFinish={onFinish}
                 treeData={treeData}
+                refresh={refresh}
                 {...props}
             />
         }