You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
434 lines
15 KiB
434 lines
15 KiB
'use strict';
|
|
|
|
const superagent = require('superagent');
|
|
|
|
const { ensureObjectPayload, ensureNonEmptyString } = require('./helpers');
|
|
const { queryClickHouseRows } = require('./clickhouse');
|
|
const {
|
|
quoteIdentifier,
|
|
validateStructuredQueryDslAgainstFields,
|
|
buildStructuredPreviewSql,
|
|
} = require('./structuredQuery');
|
|
|
|
const DATAEASE_TOKEN_CACHE_KEY = '__reportDataEaseAuth';
|
|
const DATAEASE_SUBQUERY_ALIAS = 'de_source';
|
|
|
|
const ensureDataEaseDatasetId = (value, label = 'table') => {
|
|
return ensureNonEmptyString(value, label);
|
|
};
|
|
|
|
const ensureDataEaseFieldName = (value, label = 'field') => {
|
|
return ensureNonEmptyString(value, label);
|
|
};
|
|
|
|
const getDataEaseConfig = ctx => {
|
|
const config = ctx.app?.fs?.config?.dataease || {};
|
|
const apiUrl = String(config.apiUrl || '').trim().replace(/\/+$/, '');
|
|
const username = String(config.username || '').trim();
|
|
const password = String(config.password || '').trim();
|
|
if (!apiUrl || !username || !password) {
|
|
throw 'dataease config is incomplete';
|
|
}
|
|
return { apiUrl, username, password };
|
|
};
|
|
|
|
const getDataEaseTokenCache = app => {
|
|
app.fs = app.fs || {};
|
|
app.fs[DATAEASE_TOKEN_CACHE_KEY] = app.fs[DATAEASE_TOKEN_CACHE_KEY] || {
|
|
token: '',
|
|
loginPromise: null,
|
|
};
|
|
return app.fs[DATAEASE_TOKEN_CACHE_KEY];
|
|
};
|
|
|
|
const resetDataEaseToken = app => {
|
|
const cache = getDataEaseTokenCache(app);
|
|
cache.token = '';
|
|
cache.loginPromise = null;
|
|
};
|
|
|
|
const loginDataEase = async ctx => {
|
|
const { apiUrl, username, password } = getDataEaseConfig(ctx);
|
|
const response = await superagent
|
|
.post(`${apiUrl}/de2api/login/localLogin`)
|
|
.send({ name: username, pwd: password })
|
|
.set('Content-Type', 'application/json');
|
|
|
|
const token = String(response.body?.data?.token || '').trim();
|
|
if (!token) {
|
|
throw response.body?.msg || 'dataease login failed';
|
|
}
|
|
return token;
|
|
};
|
|
|
|
const getDataEaseToken = async ctx => {
|
|
const cache = getDataEaseTokenCache(ctx.app);
|
|
if (cache.token) return cache.token;
|
|
if (!cache.loginPromise) {
|
|
cache.loginPromise = loginDataEase(ctx)
|
|
.then(token => {
|
|
cache.token = token;
|
|
cache.loginPromise = null;
|
|
return token;
|
|
})
|
|
.catch(error => {
|
|
cache.loginPromise = null;
|
|
throw error;
|
|
});
|
|
}
|
|
return cache.loginPromise;
|
|
};
|
|
|
|
const requestDataEase = async (ctx, method, path, body = undefined) => {
|
|
const { apiUrl } = getDataEaseConfig(ctx);
|
|
const execute = async token => {
|
|
let request = superagent[method](`${apiUrl}${path}`)
|
|
.set('X-DE-TOKEN', token)
|
|
.set('Content-Type', 'application/json');
|
|
|
|
if (body !== undefined) {
|
|
request = request.send(body);
|
|
}
|
|
|
|
return request;
|
|
};
|
|
|
|
const firstToken = await getDataEaseToken(ctx);
|
|
try {
|
|
const response = await execute(firstToken);
|
|
if (Number(response.body?.code) !== 0) {
|
|
throw response.body?.msg || `dataease request failed: ${path}`;
|
|
}
|
|
return response.body?.data;
|
|
} catch (error) {
|
|
const errorText = String(error?.response?.body?.msg || error?.message || error || '').toLowerCase();
|
|
const isAuthError = error?.status === 401 || errorText.includes('token');
|
|
if (!isAuthError) throw error;
|
|
|
|
resetDataEaseToken(ctx.app);
|
|
const nextToken = await getDataEaseToken(ctx);
|
|
const retryResponse = await execute(nextToken);
|
|
if (Number(retryResponse.body?.code) !== 0) {
|
|
throw retryResponse.body?.msg || `dataease request failed: ${path}`;
|
|
}
|
|
return retryResponse.body?.data;
|
|
}
|
|
};
|
|
|
|
const flattenDataEaseDatasets = (nodes = [], ancestors = []) => {
|
|
const result = [];
|
|
for (const node of nodes || []) {
|
|
const currentPath = [...ancestors, String(node?.name || '').trim()].filter(Boolean);
|
|
if (node?.leaf) {
|
|
result.push({
|
|
id: String(node.id || ''),
|
|
name: String(node.name || ''),
|
|
path: currentPath.join('/'),
|
|
leaf: true,
|
|
});
|
|
continue;
|
|
}
|
|
result.push(...flattenDataEaseDatasets(node?.children || [], currentPath));
|
|
}
|
|
return result;
|
|
};
|
|
|
|
const getDataEaseDatasetTree = async ctx => {
|
|
return requestDataEase(ctx, 'post', '/de2api/datasetTree/tree', { busiFlag: 'dataset' });
|
|
};
|
|
|
|
const getDataEaseRootChildren = async ctx => {
|
|
const tree = await getDataEaseDatasetTree(ctx);
|
|
const rootNodes = Array.isArray(tree) ? tree : [];
|
|
const rootNode = rootNodes.find(node => String(node?.id || '').trim() === '0')
|
|
|| rootNodes.find(node => String(node?.name || '').trim().toLowerCase() === 'root')
|
|
|| null;
|
|
if (Array.isArray(rootNode?.children)) return rootNode.children;
|
|
return rootNodes;
|
|
};
|
|
|
|
const resolveDataEaseRootFolder = async (ctx, folderId) => {
|
|
const normalizedFolderId = ensureNonEmptyString(folderId, 'connectionConfig.folderId');
|
|
const rootChildren = await getDataEaseRootChildren(ctx);
|
|
const folderNode = rootChildren.find(node => String(node?.id || '').trim() === normalizedFolderId);
|
|
if (!folderNode) throw `invalid param: connectionConfig.folderId (${normalizedFolderId})`;
|
|
if (folderNode?.leaf) throw 'invalid param: connectionConfig.folderId must reference a root folder';
|
|
return {
|
|
folderId: normalizedFolderId,
|
|
folderName: String(folderNode?.name || '').trim() || normalizedFolderId,
|
|
folderNode,
|
|
};
|
|
};
|
|
|
|
const normalizeDataEaseConnectionConfig = async (ctx, rawConfig) => {
|
|
const safeConfig = ensureObjectPayload(rawConfig, 'connectionConfig');
|
|
const { folderId, folderName } = await resolveDataEaseRootFolder(ctx, safeConfig.folderId);
|
|
return {
|
|
...safeConfig,
|
|
useGlobalConfig: true,
|
|
readonly: true,
|
|
sourceCode: 'dataease',
|
|
scopeType: 'root_folder',
|
|
folderId,
|
|
folderName,
|
|
};
|
|
};
|
|
|
|
const resolveDataEaseFolderScope = async (ctx, dataSource) => {
|
|
const normalizedConfig = await normalizeDataEaseConnectionConfig(ctx, dataSource?.connectionConfig || {});
|
|
const { folderNode } = await resolveDataEaseRootFolder(ctx, normalizedConfig.folderId);
|
|
return {
|
|
...normalizedConfig,
|
|
folderNode,
|
|
};
|
|
};
|
|
|
|
const listDataEaseDatasets = async (ctx, dataSource) => {
|
|
const { folderNode } = await resolveDataEaseFolderScope(ctx, dataSource);
|
|
return flattenDataEaseDatasets(folderNode?.children || [], []);
|
|
};
|
|
|
|
const ensureDataEaseDatasetInScope = async (ctx, dataSource, datasetId) => {
|
|
const normalizedDatasetId = ensureDataEaseDatasetId(datasetId);
|
|
const scope = await resolveDataEaseFolderScope(ctx, dataSource);
|
|
const datasets = flattenDataEaseDatasets(scope.folderNode?.children || [], []);
|
|
const dataset = datasets.find(item => String(item?.id || '').trim() === normalizedDatasetId);
|
|
if (!dataset) throw 'dataset does not belong to current dataease source';
|
|
return {
|
|
scope,
|
|
dataset,
|
|
};
|
|
};
|
|
|
|
const getDataEaseDatasetBarInfo = async (ctx, datasetId) => {
|
|
return requestDataEase(ctx, 'get', `/de2api/datasetTree/barInfo/${datasetId}`);
|
|
};
|
|
|
|
const getDataEaseDatasetPreview = async (ctx, datasetId) => {
|
|
return requestDataEase(ctx, 'post', `/de2api/datasetTree/get/${datasetId}`, {});
|
|
};
|
|
|
|
const decodeBase64Twice = value => {
|
|
const raw = ensureNonEmptyString(value, 'sql');
|
|
try {
|
|
const first = Buffer.from(raw, 'base64').toString('utf8');
|
|
return Buffer.from(first, 'base64').toString('utf8');
|
|
} catch (error) {
|
|
throw 'dataease sql decode failed';
|
|
}
|
|
};
|
|
|
|
const stripTrailingLimit = sql => {
|
|
return String(sql || '')
|
|
.trim()
|
|
.replace(/;+\s*$/g, '')
|
|
.replace(/\s+LIMIT\s+\d+\s*(?:,\s*\d+)?\s*$/i, '')
|
|
.trim();
|
|
};
|
|
|
|
const extractOuterProjectionAliases = sql => {
|
|
const outerSelect = String(sql || '').match(/^SELECT\s+([\s\S]+?)\s+FROM\s+\(/i);
|
|
if (!outerSelect || !outerSelect[1]) {
|
|
throw 'dataease sql format is unsupported';
|
|
}
|
|
const aliases = Array.from(
|
|
outerSelect[1].matchAll(/\bAS\s+`([^`]+)`/ig)
|
|
).map(match => String(match[1] || '').trim()).filter(Boolean);
|
|
if (!aliases.length) {
|
|
throw 'dataease sql projection aliases not found';
|
|
}
|
|
return aliases;
|
|
};
|
|
|
|
const normalizeDataEaseFields = (fields = [], sourceAliases = []) => {
|
|
if (!Array.isArray(fields) || !fields.length) {
|
|
throw 'dataease dataset fields are empty';
|
|
}
|
|
if (fields.length !== sourceAliases.length) {
|
|
throw 'dataease dataset field count does not match sql projection count';
|
|
}
|
|
|
|
const usedNames = new Set();
|
|
return fields.map((field, index) => {
|
|
const safeField = ensureObjectPayload(field, `fields[${index}]`);
|
|
const dataeaseName = String(safeField.dataeaseName || '').trim();
|
|
if (!dataeaseName) throw `invalid dataease field metadata: missing dataeaseName at fields[${index}]`;
|
|
const name = dataeaseName;
|
|
if (usedNames.has(name)) {
|
|
throw `duplicate dataease field name (${name})`;
|
|
}
|
|
usedNames.add(name);
|
|
return {
|
|
...safeField,
|
|
name,
|
|
originName: String(safeField.originName || '').trim() || null,
|
|
displayName: String(safeField.name || '').trim()
|
|
|| String(safeField.originName || '').trim()
|
|
|| name,
|
|
dataeaseName,
|
|
sourceAlias: sourceAliases[index],
|
|
position: index + 1,
|
|
};
|
|
});
|
|
};
|
|
|
|
const buildNormalizedBaseSql = fields => {
|
|
const selectSql = fields.map(field => (
|
|
`${DATAEASE_SUBQUERY_ALIAS}.${quoteIdentifier(field.sourceAlias)} AS ${quoteIdentifier(field.name)}`
|
|
)).join(', ');
|
|
return `SELECT ${selectSql} FROM (__BASE_SQL__) AS ${DATAEASE_SUBQUERY_ALIAS}`;
|
|
};
|
|
|
|
const resolveDataEaseClientContext = (ctx, barInfo = {}) => {
|
|
if (barInfo?.isCross) {
|
|
throw 'dataease cross dataset is not supported';
|
|
}
|
|
|
|
const dtoList = Array.isArray(barInfo?.datasourceDTOList) ? barInfo.datasourceDTOList : [];
|
|
if (!dtoList.length) {
|
|
throw 'dataease dataset datasource is empty';
|
|
}
|
|
|
|
const clientKeys = dtoList.map(item => {
|
|
const type = String(item?.type || '').trim().toLowerCase();
|
|
if (type !== 'ck') {
|
|
throw 'only clickhouse-backed dataease dataset is supported';
|
|
}
|
|
return ensureNonEmptyString(item?.name, 'dataease datasource name');
|
|
});
|
|
|
|
const uniqueClientKeys = Array.from(new Set(clientKeys));
|
|
if (uniqueClientKeys.length !== 1) {
|
|
throw 'dataease dataset must map to exactly one clickhouse client';
|
|
}
|
|
|
|
const clientKey = uniqueClientKeys[0];
|
|
const configuredDbs = Array.isArray(ctx.app?.fs?.config?.clickHouse?.db)
|
|
? ctx.app.fs.config.clickHouse.db
|
|
: [];
|
|
const matched = configuredDbs.find(item => String(item?.name || '').trim() === clientKey);
|
|
if (!matched) throw `clickhouse client not configured: ${clientKey}`;
|
|
|
|
const client = ctx.app?.fs?.clickHouse?.[clientKey];
|
|
if (!client) throw `clickhouse client not found: ${clientKey}`;
|
|
|
|
return {
|
|
clientKey,
|
|
database: String(matched.db || '').trim(),
|
|
client,
|
|
};
|
|
};
|
|
|
|
const resolveDataEaseDatasetContext = async (ctx, dataSource, datasetId) => {
|
|
const { scope, dataset } = await ensureDataEaseDatasetInScope(ctx, dataSource, datasetId);
|
|
const normalizedDatasetId = ensureDataEaseDatasetId(datasetId);
|
|
const [barInfo, preview] = await Promise.all([
|
|
getDataEaseDatasetBarInfo(ctx, normalizedDatasetId),
|
|
getDataEaseDatasetPreview(ctx, normalizedDatasetId),
|
|
]);
|
|
|
|
const { clientKey, database, client } = resolveDataEaseClientContext(ctx, barInfo);
|
|
const baseSql = stripTrailingLimit(decodeBase64Twice(preview?.sql));
|
|
const projectionAliases = extractOuterProjectionAliases(baseSql);
|
|
const rawFields = Array.isArray(preview?.data?.fields) ? preview.data.fields : [];
|
|
const fields = normalizeDataEaseFields(rawFields, projectionAliases);
|
|
const fieldSet = new Set(fields.map(field => field.name));
|
|
const normalizedBaseSqlTemplate = buildNormalizedBaseSql(fields);
|
|
const normalizedBaseSql = normalizedBaseSqlTemplate.replace('__BASE_SQL__', baseSql);
|
|
|
|
return {
|
|
datasetId: normalizedDatasetId,
|
|
datasetName: String(preview?.name || barInfo?.name || dataset?.name || normalizedDatasetId),
|
|
folderId: scope.folderId,
|
|
folderName: scope.folderName,
|
|
clientKey,
|
|
database,
|
|
client,
|
|
baseSql,
|
|
normalizedBaseSql,
|
|
fields,
|
|
fieldSet,
|
|
previewRows: Array.isArray(preview?.data?.data) ? preview.data.data : [],
|
|
previewTotal: preview?.total ?? null,
|
|
};
|
|
};
|
|
|
|
const getDataEaseDatasetColumns = async (ctx, dataSource, datasetId) => {
|
|
const context = await resolveDataEaseDatasetContext(ctx, dataSource, datasetId);
|
|
return {
|
|
...context,
|
|
rows: context.fields.map(field => ({
|
|
name: field.name,
|
|
type: String(field.type || ''),
|
|
originName: field.originName || null,
|
|
displayName: field.displayName,
|
|
dataeaseName: field.dataeaseName || null,
|
|
fieldShortName: field.fieldShortName || null,
|
|
groupType: field.groupType || null,
|
|
position: field.position,
|
|
})),
|
|
};
|
|
};
|
|
|
|
const validateDataEaseStructuredQueryDsl = async (ctx, dataSource, rawDsl) => {
|
|
const datasetId = ensureDataEaseDatasetId(rawDsl?.table, 'table');
|
|
const context = await resolveDataEaseDatasetContext(ctx, dataSource, datasetId);
|
|
const normalizedDsl = validateStructuredQueryDslAgainstFields(rawDsl, context.fieldSet, {
|
|
tableParser: ensureDataEaseDatasetId,
|
|
fieldParser: ensureDataEaseFieldName,
|
|
aliasParser: ensureDataEaseFieldName,
|
|
});
|
|
return {
|
|
context,
|
|
normalizedDsl,
|
|
};
|
|
};
|
|
|
|
const previewDataEaseQuery = async (ctx, dataSource, rawDsl) => {
|
|
const { context, normalizedDsl } = await validateDataEaseStructuredQueryDsl(ctx, dataSource, rawDsl);
|
|
const queryBundle = buildStructuredPreviewSql(normalizedDsl, {
|
|
tableSql: `(${context.normalizedBaseSql}) AS de_dataset`,
|
|
});
|
|
|
|
const rows = await queryClickHouseRows(
|
|
context.client,
|
|
queryBundle.query,
|
|
queryBundle.queryParams
|
|
);
|
|
|
|
let total = null;
|
|
if (!queryBundle.hasGroupBy) {
|
|
const totalSql = `SELECT count() AS total FROM ${queryBundle.tableSql}${queryBundle.whereClause} FORMAT JSON`;
|
|
const totalRows = await queryClickHouseRows(
|
|
context.client,
|
|
totalSql,
|
|
queryBundle.queryParams
|
|
);
|
|
if (Array.isArray(totalRows) && totalRows.length && totalRows[0]?.total !== undefined) {
|
|
total = Number(totalRows[0].total);
|
|
}
|
|
}
|
|
|
|
return {
|
|
sourceType: 'dataease',
|
|
dataSourceId: null,
|
|
datasetId: context.datasetId,
|
|
datasetName: context.datasetName,
|
|
clientKey: context.clientKey,
|
|
database: context.database,
|
|
columns: queryBundle.resultColumns,
|
|
rows,
|
|
total,
|
|
sqlPreview: queryBundle.sqlPreview,
|
|
baseSqlPreview: context.baseSql,
|
|
};
|
|
};
|
|
|
|
module.exports = {
|
|
ensureDataEaseDatasetId,
|
|
normalizeDataEaseConnectionConfig,
|
|
listDataEaseDatasets,
|
|
getDataEaseDatasetColumns,
|
|
validateDataEaseStructuredQueryDsl,
|
|
previewDataEaseQuery,
|
|
};
|
|
|