diff --git a/api/app/lib/controllers/metadataAcquisition/dataSource.js b/api/app/lib/controllers/metadataAcquisition/dataSource.js index 23c8d7b..bc740af 100644 --- a/api/app/lib/controllers/metadataAcquisition/dataSource.js +++ b/api/app/lib/controllers/metadataAcquisition/dataSource.js @@ -116,11 +116,20 @@ function deleteDataSource(opts) { ctx.status = 400; ctx.body = { message: '数据源下存在采集任务，请删除任务后再删除数据源！' } } else { + const datasource = await models.DataSource.findOne({ where: { id: id } }) await models.DataSource.destroy({ where: { id: id } }) + + //删除数据源绑定的资源目录下的元数据 + await models.MetadataDatabase.destroy({ + where: { + catalog: datasource.mountPath, + } + }) + ctx.status = 204; ctx.body = { message: '删除数据源成功' } } diff --git a/api/app/lib/controllers/metadataAcquisition/initJob.js b/api/app/lib/controllers/metadataAcquisition/initJob.js index 92f8d06..13a361f 100644 --- a/api/app/lib/controllers/metadataAcquisition/initJob.js +++ b/api/app/lib/controllers/metadataAcquisition/initJob.js @@ -3,9 +3,11 @@ const { handleTask } = require('./taskHandle'); //根据任务配置生成定时任务 module.exports = async function (app, task) { - let job = null + const { models } = app.fs.dc + const startTime = moment() try { + job = app.fs.scheduleInit( { interval: task.cron, @@ -15,13 +17,21 @@ module.exports = async function (app, task) { async () => { try { await handleTask(app, task); - console.log(task.taskName, moment().format('YYYY-MM-DD HH:mm:ss')); } catch (error) { app.fs.logger.error(`sechedule: taskJobs, error: ${error}`) } }); } catch (error) { + const endTime = moment() + const logBody = { + task: task.id, + success: true, + details: '采集失败' + JSON.stringify(error).substring(0, 248), + startTime: startTime, + endTime: endTime + } + await models.AcquisitionLog.create(logBody) app.fs.logger.error(`sechedule: taskJobs, error: ${error}`); } diff --git a/api/app/lib/controllers/metadataAcquisition/taskHandle.js b/api/app/lib/controllers/metadataAcquisition/taskHandle.js index 8e56887..1a3bb64 100644 --- a/api/app/lib/controllers/metadataAcquisition/taskHandle.js +++ b/api/app/lib/controllers/metadataAcquisition/taskHandle.js @@ -1,12 +1,16 @@ const Automate = require('sequelize-automate-freesun') const moment = require('moment'); -//@toto删除数据库源时 同步删除该数据源资源目录树下元数据 +const schedule = require('node-schedule') +const initJob = require('./initJob'); +//删除数据库源时 同步删除该数据源资源目录树下元数据 //数据源 数据库配置信息不能更改 若需更改则需要删除数据源重新配置 +let i = 0; async function handleTask(app, task) { - const transaction = await app.fs.dc.orm.transaction(); + // const transaction = await app.fs.dc.orm.transaction(); + const startTime = moment() + const { models } = app.fs.dc try { - const { models } = app.fs.dc const dataSource = await models.DataSource.findOne({ where: { id: task.dataSourceId @@ -16,7 +20,7 @@ async function handleTask(app, task) { if (dataSource) { const dbOptions = createDbOptions(dataSource.config); const automate = new Automate(dbOptions, {}); - const tables = await automate.getTables(); + const tables = await automate.getTables(); //获取当前采集任务数据源pg库表字段索引外键数据 const dataToSave = { code: dataSource.config.database, name: dataSource.config.database, @@ -29,7 +33,7 @@ async function handleTask(app, task) { updateAt: null, user: { id: 1, name: "超级管理员", username: "SuperAdmin" }, attributesParam: null, - catalogKey: 'rc' + catalogKey: dataSource.catalogKey } //初始化数据 新增 数据库 表 字段索引外键数据 @@ -40,7 +44,7 @@ async function handleTask(app, task) { catalog: dataSource.mountPath, }//目录树下库只会存在一个 判断是否有库类型元数据 没有则新增库和表元数据 有库数据则比较更新表类型元数据 全量更新字段索引数据 }) - + let databaseId = databaseFind ? databaseFind.id : null; if (databaseFind) { //更新表类型元数据 const newTableNames = [] Object.keys(tables).forEach(key => { newTableNames.push(key) }); @@ -78,6 +82,7 @@ async function handleTask(app, task) { } else { //新增库、表类型数据 //库类型元数据存储 const databaseRslt = await models.MetadataDatabase.create(dataToSave) + databaseId = databaseRslt.id //表元数据存储 if (databaseRslt && databaseRslt.id) { const tableBodys = [] @@ -93,43 +98,84 @@ async function handleTask(app, task) { } } - const metaDatabaseTables = await models.MetadataDatabase.findAll({ - where: { - type: '表', - catalog: dataSource.mountPath, - } - }) + if (databaseId) { + const metaDatabaseTables = await models.MetadataDatabase.findAll({ + where: { + type: '表', + catalog: dataSource.mountPath, + parent: databaseId + } + }) - //字段/索引/外键全量更新 先删除之前的字段 再录入新的数据 - await models.MetadataDatabase.destroy({ - where: { - type: { $in: ['字段', '索引', '外键'] }, - catalog: dataSource.mountPath, + //字段/索引/外键全量更新 先删除之前的字段 再录入新的数据 + await models.MetadataDatabase.destroy({ + where: { + type: { $in: ['字段', '索引', '外键'] }, + catalog: dataSource.mountPath, + } + }) + + const fieldBodys = [] + for (let table of metaDatabaseTables) { + Object.keys(tables[table.name].structures).forEach(key => { + dataToSave.parent = table.id; + dataToSave.name = tables[table.name].structures[key].comment || key; + dataToSave.code = key; + dataToSave.type = '字段'; + const tableObj = { ...dataToSave } + fieldBodys.push(tableObj) + }) + + tables[table.name].foreignKeys.forEach(v => { + dataToSave.parent = table.id; + dataToSave.name = v.columnName; + dataToSave.code = v.columnName; + dataToSave.type = '外键'; + const tableObj = { ...dataToSave } + fieldBodys.push(tableObj) + }) + + tables[table.name].indexes.forEach(v => { + dataToSave.parent = table.id; + dataToSave.name = v.name; + dataToSave.code = v.name; + dataToSave.type = '索引'; + const tableObj = { ...dataToSave } + fieldBodys.push(tableObj) + }) } - }) - const fieldBodys = [] - for (let table of metaDatabaseTables) { - if(!tables[table.name].structures){ - console.log(table) - return; + const fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys); + if (fieldRslt) { + const endTime = moment() + const logBody = { + task: task.id, + success: true, + details: '采集成功', + startTime: startTime, + endTime: endTime + + } + //日志记录 + await models.AcquisitionLog.create(logBody) } - Object.keys(tables[table.name].structures).forEach(key => { - dataToSave.parent = table.id; - dataToSave.name = tables[table.name].structures[key].comment || key; - dataToSave.code = key; - dataToSave.type = '字段'; - const tableObj = { ...dataToSave } - fieldBodys.push(tableObj) - }) - } - const fieldRslt = await models.MetadataDatabase.bulkCreate(fieldBodys); - if (fieldRslt) { - //日志记录 } } + i++ + console.log('===========------------------------------------------------------------' + task.taskName + moment().format('HH:mm:ss')) } catch (error) { // await transaction.rollback(); + const endTime = moment() + const logBody = { + task: task.id, + success: true, + details: '采集失败' + JSON.stringify(error).substring(0, 248), + startTime: startTime, + endTime: endTime + } + await models.AcquisitionLog.create(logBody) + // if (app.fs.schedule['taskJobs'][task.id]) schedule.cancelJob(app.fs.schedule['taskJobs'][task.id]) + console.log('===========------------------------------------------------------------' + i) app.fs.logger.error(`sechedule: handleTask, error: ${error}`); } } diff --git a/api/app/lib/models/data_source.js b/api/app/lib/models/data_source.js index d5b7703..c86538a 100644 --- a/api/app/lib/models/data_source.js +++ b/api/app/lib/models/data_source.js @@ -86,6 +86,15 @@ module.exports = dc => { primaryKey: false, field: "time", autoIncrement: false + }, + catalogKey: { + type: DataTypes.STRING, + allowNull: true, + defaultValue: null, + comment: null, + primaryKey: false, + field: "catalogKey", + autoIncrement: false } }, { tableName: "t_data_source", diff --git a/scripts/0.0.3/schema/03_alter_t_data_source.sql b/scripts/0.0.3/schema/03_alter_t_data_source.sql new file mode 100644 index 0000000..c8c9e2d --- /dev/null +++ b/scripts/0.0.3/schema/03_alter_t_data_source.sql @@ -0,0 +1,2 @@ +alter table t_data_source + add "catalogKey" varchar(255); \ No newline at end of file diff --git a/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js b/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js index 26211f6..a3fb950 100644 --- a/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js +++ b/web/client/src/sections/metadataAcquisition/components/steps/postgre/stepOne.js @@ -10,7 +10,7 @@ import { import '../../style.less'; function StepOne(props) { - const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData } = props; + const { next, stepOneValues, stepOneValuesFinish, readOnly, treeData, dataSources } = props; const formRef = React.createRef(); const initialValues = stepOneValues ? stepOneValues : { adapterName: 'PostgreSQL采集适配器', @@ -30,7 +30,10 @@ function StepOne(props) { data = dataSource.filter(ds => ds.parent == parent); } treeData = data.map(ds => { - return { title: ds.name, key: ds?.id, id: ds.id, value: ds?.id } + return { + title: ds.name, key: `${key}-${ds.id}`, id: ds.id, value: `${key}-${ds.id}`, + disabled: dataSources?.rows?.find(s => s.mountPath == ds.id) + } }); for (let d of treeData) { d.children = getTreeNodeData(dataSource, d.id, d.key); @@ -50,6 +53,7 @@ function StepOne(props) { onCancel: () => { }, }} onFinish={async (values) => { + values.mountPath = values.catalogKey.split('-')[values.catalogKey.split('-')?.length - 1] next() stepOneValuesFinish(values) return true; @@ -109,7 +113,7 @@ function StepOne(props) { // disabled={true} /> */} { return { const obj = {}; @@ -88,6 +89,7 @@ function StepTwo(props) { return true; } } + const checkConnect = () => { if (checkNext() === true) { setLoading(true) diff --git a/web/client/src/sections/metadataAcquisition/containers/acquisitionTask.js b/web/client/src/sections/metadataAcquisition/containers/acquisitionTask.js index eb893c4..53b9f38 100644 --- a/web/client/src/sections/metadataAcquisition/containers/acquisitionTask.js +++ b/web/client/src/sections/metadataAcquisition/containers/acquisitionTask.js @@ -122,9 +122,9 @@ function AcquisitionTask(props) { { title: '执行周期', dataIndex: 'cron', - // render: (text, record) => { - // return transCron(record?.cron); - // } + render: (text, record) => { + return transCron(record?.cron); + } }, { title: '重复次数', diff --git a/web/client/src/sections/metadataAcquisition/containers/adapter.js b/web/client/src/sections/metadataAcquisition/containers/adapter.js index ff0c659..9d533a8 100644 --- a/web/client/src/sections/metadataAcquisition/containers/adapter.js +++ b/web/client/src/sections/metadataAcquisition/containers/adapter.js @@ -13,6 +13,7 @@ const LatestMetadata = (props) => { const { data: treeData = [] } = useFsRequest({ url: ApiTable.getResourceCatalog }); useEffect(() => { dispatch(actions.metadataAcquisition.getAdapters()) + dispatch(actions.metadataAcquisition.getDataSources()); }, []) const renderRelationalDatabase = () => { @@ -42,7 +43,8 @@ const LatestMetadata = (props) => { name: stepOneValues?.name, audited: true, adapterId: adapterInfo?.id, - mountPath: 1, + mountPath: stepOneValues?.mountPath, + catalogKey: stepOneValues?.catalogKey, description: stepOneValues?.description, config: stepTwoValues, time: moment() @@ -91,12 +93,13 @@ const LatestMetadata = (props) => { } function mapStateToProps(state) { - const { global, auth, adapters } = state; + const { global, auth, adapters, datasources } = state; return { clientHeight: global.clientHeight, user: auth.user, actions: global.actions, - adapters: adapters?.data || [] + adapters: adapters?.data || [], + dataSources: datasources?.data || {}, }; } export default connect(mapStateToProps)(LatestMetadata) \ No newline at end of file diff --git a/web/client/src/sections/metadataAcquisition/containers/dataSourceManagement.js b/web/client/src/sections/metadataAcquisition/containers/dataSourceManagement.js index 5367fbc..9935991 100644 --- a/web/client/src/sections/metadataAcquisition/containers/dataSourceManagement.js +++ b/web/client/src/sections/metadataAcquisition/containers/dataSourceManagement.js @@ -94,7 +94,8 @@ function DataSourceManagement(props) {
是否确认删除该数据源？
+
(将同步删除数据源下的元数据)
} onConfirm={() => handleDelete(record.id)} okText="是" cancelText="否" @@ -116,6 +117,7 @@ function DataSourceManagement(props) { audited: true, adapterId: adapterInfo?.id, mountPath: stepOneValues?.mountPath, + catalogKey: stepOneValues?.catalogKey, description: stepOneValues?.description, config: stepTwoValues, time: moment()