(*)元数据定时任务功能完善 数据源增加资源目录树key

api/app/lib/controllers/metadataAcquisition/dataSource.js

@@ -116,11 +116,20 @@ function deleteDataSource(opts) {
                 ctx.status = 400;
                 ctx.body = { message: '数据源下存在采集任务，请删除任务后再删除数据源！' }
             } else {
+                const datasource = await models.DataSource.findOne({ where: { id: id } })
                 await models.DataSource.destroy({
                     where: {
                         id: id
                     }
                 })
+
+                //删除数据源绑定的资源目录下的元数据
+                await models.MetadataDatabase.destroy({
+                    where: {
+                        catalog: datasource.mountPath,
+                    }
+                })
+
                 ctx.status = 204;
                 ctx.body = { message: '删除数据源成功' }
             }

api/app/lib/controllers/metadataAcquisition/initJob.js

@@ -3,9 +3,11 @@ const { handleTask } = require('./taskHandle');
 
 //根据任务配置生成定时任务
 module.exports = async function (app, task) {
-
     let job = null
+    const { models } = app.fs.dc
+    const startTime = moment()
     try {
+
         job = app.fs.scheduleInit(
             {
                 interval: task.cron,
@@ -15,13 +17,21 @@ module.exports = async function (app, task) {
             async () => {
                 try {
                     await handleTask(app, task);
-                    console.log(task.taskName, moment().format('YYYY-MM-DD HH:mm:ss'));
                 } catch (error) {
                     app.fs.logger.error(`sechedule: taskJobs, error: ${error}`)
                 }
             });
 
     } catch (error) {
+        const endTime = moment()
+        const logBody = {
+            task: task.id,
+            success: true,
+            details: '采集失败' + JSON.stringify(error).substring(0, 248),
+            startTime: startTime,
+            endTime: endTime
+        }
+        await models.AcquisitionLog.create(logBody)
         app.fs.logger.error(`sechedule: taskJobs, error: ${error}`);
     }
 

api/app/lib/controllers/metadataAcquisition/taskHandle.js

@@ -1,12 +1,16 @@
 
 const Automate = require('sequelize-automate-freesun')
 const moment = require('moment');
-//@toto删除数据库源时 同步删除该数据源资源目录树下元数据 
+const schedule = require('node-schedule')
+const initJob = require('./initJob');
+//删除数据库源时 同步删除该数据源资源目录树下元数据 
 //数据源 数据库配置信息不能更改 若需更改则需要删除数据源重新配置
+let i = 0;
 async function handleTask(app, task) {
-    const transaction = await app.fs.dc.orm.transaction();
+    // const transaction = await app.fs.dc.orm.transaction();
+    const startTime = moment()
+    const { models } = app.fs.dc
     try {
-        const { models } = app.fs.dc
         const dataSource = await models.DataSource.findOne({
             where: {
                 id: task.dataSourceId
@@ -16,7 +20,7 @@ async function handleTask(app, task) {
         if (dataSource) {
             const dbOptions = createDbOptions(dataSource.config);
             const automate = new Automate(dbOptions, {});
-            const tables = await automate.getTables();
+            const tables = await automate.getTables(); //获取当前采集任务数据源pg库表字段索引外键数据
             const dataToSave = {
                 code: dataSource.config.database,
                 name: dataSource.config.database,
@@ -29,7 +33,7 @@ async function handleTask(app, task) {
                 updateAt: null,
                 user: { id: 1, name: "超级管理员", username: "SuperAdmin" },
                 attributesParam: null,
-                catalogKey: 'rc'
+                catalogKey: dataSource.catalogKey
             }
 
             //初始化数据 新增 数据库 表 字段索引外键数据
@@ -40,7 +44,7 @@ async function handleTask(app, task) {
                     catalog: dataSource.mountPath,
                 }//目录树下库只会存在一个 判断是否有库类型元数据 没有则新增库和表元数据 有库数据则比较更新表类型元数据 全量更新字段索引数据
             })
-
+            let databaseId = databaseFind ? databaseFind.id : null;
             if (databaseFind) { //更新表类型元数据
                 const newTableNames = []
                 Object.keys(tables).forEach(key => { newTableNames.push(key) });
@@ -78,6 +82,7 @@ async function handleTask(app, task) {
             } else { //新增库、表类型数据
                 //库类型元数据存储
                 const databaseRslt = await models.MetadataDatabase.create(dataToSave)
+                databaseId = databaseRslt.id
                 //表元数据存储
                 if (databaseRslt && databaseRslt.id) {
                     const tableBodys = []
@@ -93,43 +98,84 @@ async function handleTask(app, task) {
                 }
             }
 
-            const metaDatabaseTables = await models.MetadataDatabase.findAll({
-                where: {
-                    type: '表',
-                    catalog: dataSource.mountPath,
-                }
-            })
+            if (databaseId) {
+                const metaDatabaseTables = await models.MetadataDatabase.findAll({
+                    where: {
+                        type: '表',
+                        catalog: dataSource.mountPath,
+                        parent: databaseId
+                    }
+                })
 
-            //字段/索引/外键全量更新 先删除之前的字段 再录入新的数据
-            await models.MetadataDatabase.destroy({
-                where: {
-                    type: { $in: ['字段', '索引', '外键'] },
-                    catalog: dataSource.mountPath,
+                //字段/索引/外键全量更新 先删除之前的字段 再录入新的数据
+                await models.MetadataDatabase.destroy({
+                    where: {
+                        type: { $in: ['字段', '索引', '外键'] },
+                        catalog: dataSource.mountPath,
+                    }
+                })
+
+                const fieldBodys = []
+                for (let table of metaDatabaseTables) {
+                    Object.keys(tables[table.name].structures).forEach(key => {
+                        dataToSave.parent = table.id;
+                        dataToSave.name = tables[table.name].structures[key].comment || key;
+                        dataToSave.code = key;
+                        dataToSave.type = '字段';
+                        const tableObj = { ...dataToSave }
+                        fieldBodys.push(tableObj)
+                    })
+
+                    tables[table.name].foreignKeys.forEach(v => {
+                        dataToSave.parent = table.id;
+                        dataToSave.name = v.columnName;
+                        dataToSave.code = v.columnName;
+                        dataToSave.type = '外键';
+                        const tableObj = { ...dataToSave }
+                        fieldBodys.push(tableObj)
+                    })
+
+                    tables[table.name].indexes.forEach(v => {
+                        dataToSave.parent = table.id;
+                        dataToSave.name = v.name;
+                        dataToSave.code = v.name;
+                        dataToSave.type = '索引';
+                        const tableObj = { ...dataToSave }
+                        fieldBodys.push(tableObj)
+                    })
                 }
-            })
 
-            const fieldBodys = []
-            for (let table of metaDatabaseTables) {
-                if(!tables[table.name].structures){
-                    console.log(table)
-                    return;
+                const fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys);
+                if (fieldRslt) {
+                    const endTime = moment()
+                    const logBody = {
+                        task: task.id,
+                        success: true,
+                        details: '采集成功',
+                        startTime: startTime,
+                        endTime: endTime
+
+                    }
+                    //日志记录
+                    await models.AcquisitionLog.create(logBody)
                 }
-                Object.keys(tables[table.name].structures).forEach(key => {
-                    dataToSave.parent = table.id;
-                    dataToSave.name = tables[table.name].structures[key].comment || key;
-                    dataToSave.code = key;
-                    dataToSave.type = '字段';
-                    const tableObj = { ...dataToSave }
-                    fieldBodys.push(tableObj)
-                })
-            }
-            const fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys);
-            if (fieldRslt) {
-                //日志记录
             }
         }
+        i++
+        console.log('===========------------------------------------------------------------' + task.taskName + moment().format('HH:mm:ss'))
     } catch (error) {
         // await transaction.rollback();
+        const endTime = moment()
+        const logBody = {
+            task: task.id,
+            success: true,
+            details: '采集失败' + JSON.stringify(error).substring(0, 248),
+            startTime: startTime,
+            endTime: endTime
+        }
+        await models.AcquisitionLog.create(logBody)
+        // if (app.fs.schedule['taskJobs'][task.id]) schedule.cancelJob(app.fs.schedule['taskJobs'][task.id])
+        console.log('===========------------------------------------------------------------' + i)
         app.fs.logger.error(`sechedule: handleTask, error: ${error}`);
     }
 }

api/app/lib/models/data_source.js

@@ -86,6 +86,15 @@ module.exports = dc => {
             primaryKey: false,
             field: "time",
             autoIncrement: false
+        },
+        catalogKey: {
+            type: DataTypes.STRING,
+            allowNull: true,
+            defaultValue: null,
+            comment: null,
+            primaryKey: false,
+            field: "catalogKey",
+            autoIncrement: false
         }
     }, {
         tableName: "t_data_source",

scripts/0.0.3/schema/03_alter_t_data_source.sql

@@ -0,0 +1,2 @@
+alter table t_data_source
+	add "catalogKey" varchar(255);

web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js

@@ -10,7 +10,7 @@ import {
 
 import '../../style.less';
 function StepOne(props) {
-    const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData } = props;
+    const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData, dataSources } = props;
     const formRef = React.createRef();
     const initialValues = stepOneValues ? stepOneValues : {
         adapterName: 'PostgreSQL采集适配器',
@@ -30,7 +30,10 @@ function StepOne(props) {
             data = dataSource.filter(ds => ds.parent == parent);
         }
         treeData = data.map(ds => {
-            return { title: ds.name, key: ds?.id, id: ds.id, value: ds?.id }
+            return {
+                title: ds.name, key: `${key}-${ds.id}`, id: ds.id, value: `${key}-${ds.id}`,
+                disabled: dataSources?.rows?.find(s => s.mountPath == ds.id)
+            }
         });
         for (let d of treeData) {
             d.children = getTreeNodeData(dataSource, d.id, d.key);
@@ -50,6 +53,7 @@ function StepOne(props) {
                 onCancel: () => { },
             }}
             onFinish={async (values) => {
+                values.mountPath = values.catalogKey.split('-')[values.catalogKey.split('-')?.length - 1]
                 next()
                 stepOneValuesFinish(values)
                 return true;
@@ -109,7 +113,7 @@ function StepOne(props) {
             // disabled={true}
             /> */}
             <ProFormTreeSelect
-                name="mountPath"
+                name="catalogKey"
                 label="数据源挂载路径"
                 placeholder="请选择数据源挂载路径"
                 rules={[{ required: true, message: '请选择数据源挂载路径' }]}

web/client/src/sections/metadataAcquisition/components/steps/postgre/stepTwo.js

@@ -4,7 +4,7 @@ import '../../style.less';
 import { Func, useMockRequest, useFsRequest, ApiTable } from '$utils';
 
 function StepTwo(props) {
-    const { prev, next, dispatch, actions, stepTwoValues, stepTwoValuesFinish, stepProps } = props;
+    const { prev, next, dispatch, actions, stepTwoValues, stepTwoValuesFinish, stepProps, editData } = props;
     const [params, setParams] = useState({})
     const [connect, setConnet] = useState('')
     const [loading, setLoading] = useState(false);
@@ -69,6 +69,7 @@ function StepTwo(props) {
             render: (text, record) => {
                 return <Input
                     // type={text == "password" ? "password" : ''}
+                    disabled={!!editData}
                     defaultValue={stepTwoValues ? stepTwoValues[text] : ''}
                     onChange={(e) => {
                         const obj = {};
@@ -88,6 +89,7 @@ function StepTwo(props) {
             return true;
         }
     }
+
     const checkConnect = () => {
         if (checkNext() === true) {
             setLoading(true)

web/client/src/sections/metadataAcquisition/containers/acquisitionTask.js

@@ -122,9 +122,9 @@ function AcquisitionTask(props) {
         {
             title: '执行周期',
             dataIndex: 'cron',
-            // render: (text, record) => {
-            //     return transCron(record?.cron);
-            // }
+            render: (text, record) => {
+                return transCron(record?.cron);
+            }
         },
         {
             title: '重复次数',

web/client/src/sections/metadataAcquisition/containers/adapter.js

@@ -13,6 +13,7 @@ const LatestMetadata = (props) => {
     const { data: treeData = [] } = useFsRequest({ url: ApiTable.getResourceCatalog });
     useEffect(() => {
         dispatch(actions.metadataAcquisition.getAdapters())
+        dispatch(actions.metadataAcquisition.getDataSources());
     }, [])
 
     const renderRelationalDatabase = () => {
@@ -42,7 +43,8 @@ const LatestMetadata = (props) => {
                 name: stepOneValues?.name,
                 audited: true,
                 adapterId: adapterInfo?.id,
-                mountPath: 1,
+                mountPath: stepOneValues?.mountPath,
+                catalogKey: stepOneValues?.catalogKey,
                 description: stepOneValues?.description,
                 config: stepTwoValues,
                 time: moment()
@@ -91,12 +93,13 @@ const LatestMetadata = (props) => {
 }
 
 function mapStateToProps(state) {
-    const { global, auth, adapters } = state;
+    const { global, auth, adapters, datasources } = state;
     return {
         clientHeight: global.clientHeight,
         user: auth.user,
         actions: global.actions,
-        adapters: adapters?.data || []
+        adapters: adapters?.data || [],
+        dataSources: datasources?.data || {},
     };
 }
 export default connect(mapStateToProps)(LatestMetadata)

web/client/src/sections/metadataAcquisition/containers/dataSourceManagement.js

@@ -94,7 +94,8 @@ function DataSourceManagement(props) {
                     <Popconfirm
                         key="del"
                         placement="top"
-                        title="是否确认删除该数据源？"
+                        title={<><div>是否确认删除该数据源？</div>
+                            <div>(将同步删除数据源下的元数据)</div></>}
                         onConfirm={() => handleDelete(record.id)}
                         okText="是"
                         cancelText="否"
@@ -116,6 +117,7 @@ function DataSourceManagement(props) {
                 audited: true,
                 adapterId: adapterInfo?.id,
                 mountPath: stepOneValues?.mountPath,
+                catalogKey: stepOneValues?.catalogKey,
                 description: stepOneValues?.description,
                 config: stepTwoValues,
                 time: moment()