'use strict'; const superagent = require('superagent'); const { ensureObjectPayload, ensureNonEmptyString } = require('./helpers'); const { queryClickHouseRows } = require('./clickhouse'); const { quoteIdentifier, validateStructuredQueryDslAgainstFields, buildStructuredPreviewSql, } = require('./structuredQuery'); const DATAEASE_TOKEN_CACHE_KEY = '__reportDataEaseAuth'; const DATAEASE_SUBQUERY_ALIAS = 'de_source'; const ensureDataEaseDatasetId = (value, label = 'table') => { return ensureNonEmptyString(value, label); }; const ensureDataEaseFieldName = (value, label = 'field') => { return ensureNonEmptyString(value, label); }; const getDataEaseConfig = ctx => { const config = ctx.app?.fs?.config?.dataease || {}; const apiUrl = String(config.apiUrl || '').trim().replace(/\/+$/, ''); const username = String(config.username || '').trim(); const password = String(config.password || '').trim(); if (!apiUrl || !username || !password) { throw 'dataease config is incomplete'; } return { apiUrl, username, password }; }; const getDataEaseTokenCache = app => { app.fs = app.fs || {}; app.fs[DATAEASE_TOKEN_CACHE_KEY] = app.fs[DATAEASE_TOKEN_CACHE_KEY] || { token: '', loginPromise: null, }; return app.fs[DATAEASE_TOKEN_CACHE_KEY]; }; const resetDataEaseToken = app => { const cache = getDataEaseTokenCache(app); cache.token = ''; cache.loginPromise = null; }; const loginDataEase = async ctx => { const { apiUrl, username, password } = getDataEaseConfig(ctx); const response = await superagent .post(`${apiUrl}/de2api/login/localLogin`) .send({ name: username, pwd: password }) .set('Content-Type', 'application/json'); const token = String(response.body?.data?.token || '').trim(); if (!token) { throw response.body?.msg || 'dataease login failed'; } return token; }; const getDataEaseToken = async ctx => { const cache = getDataEaseTokenCache(ctx.app); if (cache.token) return cache.token; if (!cache.loginPromise) { cache.loginPromise = loginDataEase(ctx) .then(token => { cache.token = token; cache.loginPromise = null; return token; }) .catch(error => { cache.loginPromise = null; throw error; }); } return cache.loginPromise; }; const requestDataEase = async (ctx, method, path, body = undefined) => { const { apiUrl } = getDataEaseConfig(ctx); const execute = async token => { let request = superagent[method](`${apiUrl}${path}`) .set('X-DE-TOKEN', token) .set('Content-Type', 'application/json'); if (body !== undefined) { request = request.send(body); } return request; }; const firstToken = await getDataEaseToken(ctx); try { const response = await execute(firstToken); if (Number(response.body?.code) !== 0) { throw response.body?.msg || `dataease request failed: ${path}`; } return response.body?.data; } catch (error) { const errorText = String(error?.response?.body?.msg || error?.message || error || '').toLowerCase(); const isAuthError = error?.status === 401 || errorText.includes('token'); if (!isAuthError) throw error; resetDataEaseToken(ctx.app); const nextToken = await getDataEaseToken(ctx); const retryResponse = await execute(nextToken); if (Number(retryResponse.body?.code) !== 0) { throw retryResponse.body?.msg || `dataease request failed: ${path}`; } return retryResponse.body?.data; } }; const flattenDataEaseDatasets = (nodes = [], ancestors = []) => { const result = []; for (const node of nodes || []) { const currentPath = [...ancestors, String(node?.name || '').trim()].filter(Boolean); if (node?.leaf) { result.push({ id: String(node.id || ''), name: String(node.name || ''), path: currentPath.join('/'), leaf: true, }); continue; } result.push(...flattenDataEaseDatasets(node?.children || [], currentPath)); } return result; }; const getDataEaseDatasetTree = async ctx => { return requestDataEase(ctx, 'post', '/de2api/datasetTree/tree', { busiFlag: 'dataset' }); }; const getDataEaseRootChildren = async ctx => { const tree = await getDataEaseDatasetTree(ctx); const rootNodes = Array.isArray(tree) ? tree : []; const rootNode = rootNodes.find(node => String(node?.id || '').trim() === '0') || rootNodes.find(node => String(node?.name || '').trim().toLowerCase() === 'root') || null; if (Array.isArray(rootNode?.children)) return rootNode.children; return rootNodes; }; const resolveDataEaseRootFolder = async (ctx, folderId) => { const normalizedFolderId = ensureNonEmptyString(folderId, 'connectionConfig.folderId'); const rootChildren = await getDataEaseRootChildren(ctx); const folderNode = rootChildren.find(node => String(node?.id || '').trim() === normalizedFolderId); if (!folderNode) throw `invalid param: connectionConfig.folderId (${normalizedFolderId})`; if (folderNode?.leaf) throw 'invalid param: connectionConfig.folderId must reference a root folder'; return { folderId: normalizedFolderId, folderName: String(folderNode?.name || '').trim() || normalizedFolderId, folderNode, }; }; const normalizeDataEaseConnectionConfig = async (ctx, rawConfig) => { const safeConfig = ensureObjectPayload(rawConfig, 'connectionConfig'); const { folderId, folderName } = await resolveDataEaseRootFolder(ctx, safeConfig.folderId); return { ...safeConfig, useGlobalConfig: true, readonly: true, sourceCode: 'dataease', scopeType: 'root_folder', folderId, folderName, }; }; const resolveDataEaseFolderScope = async (ctx, dataSource) => { const normalizedConfig = await normalizeDataEaseConnectionConfig(ctx, dataSource?.connectionConfig || {}); const { folderNode } = await resolveDataEaseRootFolder(ctx, normalizedConfig.folderId); return { ...normalizedConfig, folderNode, }; }; const listDataEaseDatasets = async (ctx, dataSource) => { const { folderNode } = await resolveDataEaseFolderScope(ctx, dataSource); return flattenDataEaseDatasets(folderNode?.children || [], []); }; const ensureDataEaseDatasetInScope = async (ctx, dataSource, datasetId) => { const normalizedDatasetId = ensureDataEaseDatasetId(datasetId); const scope = await resolveDataEaseFolderScope(ctx, dataSource); const datasets = flattenDataEaseDatasets(scope.folderNode?.children || [], []); const dataset = datasets.find(item => String(item?.id || '').trim() === normalizedDatasetId); if (!dataset) throw 'dataset does not belong to current dataease source'; return { scope, dataset, }; }; const getDataEaseDatasetBarInfo = async (ctx, datasetId) => { return requestDataEase(ctx, 'get', `/de2api/datasetTree/barInfo/${datasetId}`); }; const getDataEaseDatasetPreview = async (ctx, datasetId) => { return requestDataEase(ctx, 'post', `/de2api/datasetTree/get/${datasetId}`, {}); }; const decodeBase64Twice = value => { const raw = ensureNonEmptyString(value, 'sql'); try { const first = Buffer.from(raw, 'base64').toString('utf8'); return Buffer.from(first, 'base64').toString('utf8'); } catch (error) { throw 'dataease sql decode failed'; } }; const stripTrailingLimit = sql => { return String(sql || '') .trim() .replace(/;+\s*$/g, '') .replace(/\s+LIMIT\s+\d+\s*(?:,\s*\d+)?\s*$/i, '') .trim(); }; const extractOuterProjectionAliases = sql => { const outerSelect = String(sql || '').match(/^SELECT\s+([\s\S]+?)\s+FROM\s+\(/i); if (!outerSelect || !outerSelect[1]) { throw 'dataease sql format is unsupported'; } const aliases = Array.from( outerSelect[1].matchAll(/\bAS\s+`([^`]+)`/ig) ).map(match => String(match[1] || '').trim()).filter(Boolean); if (!aliases.length) { throw 'dataease sql projection aliases not found'; } return aliases; }; const normalizeDataEaseFields = (fields = [], sourceAliases = []) => { if (!Array.isArray(fields) || !fields.length) { throw 'dataease dataset fields are empty'; } if (fields.length !== sourceAliases.length) { throw 'dataease dataset field count does not match sql projection count'; } const usedNames = new Set(); return fields.map((field, index) => { const safeField = ensureObjectPayload(field, `fields[${index}]`); const dataeaseName = String(safeField.dataeaseName || '').trim(); if (!dataeaseName) throw `invalid dataease field metadata: missing dataeaseName at fields[${index}]`; const name = dataeaseName; if (usedNames.has(name)) { throw `duplicate dataease field name (${name})`; } usedNames.add(name); return { ...safeField, name, originName: String(safeField.originName || '').trim() || null, displayName: String(safeField.name || '').trim() || String(safeField.originName || '').trim() || name, dataeaseName, sourceAlias: sourceAliases[index], position: index + 1, }; }); }; const buildNormalizedBaseSql = fields => { const selectSql = fields.map(field => ( `${DATAEASE_SUBQUERY_ALIAS}.${quoteIdentifier(field.sourceAlias)} AS ${quoteIdentifier(field.name)}` )).join(', '); return `SELECT ${selectSql} FROM (__BASE_SQL__) AS ${DATAEASE_SUBQUERY_ALIAS}`; }; const resolveDataEaseClientContext = (ctx, barInfo = {}) => { if (barInfo?.isCross) { throw 'dataease cross dataset is not supported'; } const dtoList = Array.isArray(barInfo?.datasourceDTOList) ? barInfo.datasourceDTOList : []; if (!dtoList.length) { throw 'dataease dataset datasource is empty'; } const clientKeys = dtoList.map(item => { const type = String(item?.type || '').trim().toLowerCase(); if (type !== 'ck') { throw 'only clickhouse-backed dataease dataset is supported'; } return ensureNonEmptyString(item?.name, 'dataease datasource name'); }); const uniqueClientKeys = Array.from(new Set(clientKeys)); if (uniqueClientKeys.length !== 1) { throw 'dataease dataset must map to exactly one clickhouse client'; } const clientKey = uniqueClientKeys[0]; const configuredDbs = Array.isArray(ctx.app?.fs?.config?.clickHouse?.db) ? ctx.app.fs.config.clickHouse.db : []; const matched = configuredDbs.find(item => String(item?.name || '').trim() === clientKey); if (!matched) throw `clickhouse client not configured: ${clientKey}`; const client = ctx.app?.fs?.clickHouse?.[clientKey]; if (!client) throw `clickhouse client not found: ${clientKey}`; return { clientKey, database: String(matched.db || '').trim(), client, }; }; const resolveDataEaseDatasetContext = async (ctx, dataSource, datasetId) => { const { scope, dataset } = await ensureDataEaseDatasetInScope(ctx, dataSource, datasetId); const normalizedDatasetId = ensureDataEaseDatasetId(datasetId); const [barInfo, preview] = await Promise.all([ getDataEaseDatasetBarInfo(ctx, normalizedDatasetId), getDataEaseDatasetPreview(ctx, normalizedDatasetId), ]); const { clientKey, database, client } = resolveDataEaseClientContext(ctx, barInfo); const baseSql = stripTrailingLimit(decodeBase64Twice(preview?.sql)); const projectionAliases = extractOuterProjectionAliases(baseSql); const rawFields = Array.isArray(preview?.data?.fields) ? preview.data.fields : []; const fields = normalizeDataEaseFields(rawFields, projectionAliases); const fieldSet = new Set(fields.map(field => field.name)); const normalizedBaseSqlTemplate = buildNormalizedBaseSql(fields); const normalizedBaseSql = normalizedBaseSqlTemplate.replace('__BASE_SQL__', baseSql); return { datasetId: normalizedDatasetId, datasetName: String(preview?.name || barInfo?.name || dataset?.name || normalizedDatasetId), folderId: scope.folderId, folderName: scope.folderName, clientKey, database, client, baseSql, normalizedBaseSql, fields, fieldSet, previewRows: Array.isArray(preview?.data?.data) ? preview.data.data : [], previewTotal: preview?.total ?? null, }; }; const getDataEaseDatasetColumns = async (ctx, dataSource, datasetId) => { const context = await resolveDataEaseDatasetContext(ctx, dataSource, datasetId); return { ...context, rows: context.fields.map(field => ({ name: field.name, type: String(field.type || ''), originName: field.originName || null, displayName: field.displayName, dataeaseName: field.dataeaseName || null, fieldShortName: field.fieldShortName || null, groupType: field.groupType || null, position: field.position, })), }; }; const validateDataEaseStructuredQueryDsl = async (ctx, dataSource, rawDsl) => { const datasetId = ensureDataEaseDatasetId(rawDsl?.table, 'table'); const context = await resolveDataEaseDatasetContext(ctx, dataSource, datasetId); const normalizedDsl = validateStructuredQueryDslAgainstFields(rawDsl, context.fieldSet, { tableParser: ensureDataEaseDatasetId, fieldParser: ensureDataEaseFieldName, aliasParser: ensureDataEaseFieldName, }); return { context, normalizedDsl, }; }; const previewDataEaseQuery = async (ctx, dataSource, rawDsl) => { const { context, normalizedDsl } = await validateDataEaseStructuredQueryDsl(ctx, dataSource, rawDsl); const queryBundle = buildStructuredPreviewSql(normalizedDsl, { tableSql: `(${context.normalizedBaseSql}) AS de_dataset`, }); const rows = await queryClickHouseRows( context.client, queryBundle.query, queryBundle.queryParams ); let total = null; if (!queryBundle.hasGroupBy) { const totalSql = `SELECT count() AS total FROM ${queryBundle.tableSql}${queryBundle.whereClause} FORMAT JSON`; const totalRows = await queryClickHouseRows( context.client, totalSql, queryBundle.queryParams ); if (Array.isArray(totalRows) && totalRows.length && totalRows[0]?.total !== undefined) { total = Number(totalRows[0].total); } } return { sourceType: 'dataease', dataSourceId: null, datasetId: context.datasetId, datasetName: context.datasetName, clientKey: context.clientKey, database: context.database, columns: queryBundle.resultColumns, rows, total, sqlPreview: queryBundle.sqlPreview, baseSqlPreview: context.baseSql, }; }; module.exports = { ensureDataEaseDatasetId, normalizeDataEaseConnectionConfig, listDataEaseDatasets, getDataEaseDatasetColumns, validateDataEaseStructuredQueryDsl, previewDataEaseQuery, };