'use strict'; const { CLICKHOUSE_FILTER_OPERATORS, CLICKHOUSE_SORT_DIRECTIONS, CLICKHOUSE_AGGREGATE_FUNCTION_MAP, } = require('./constants'); const { hasOwn, toInt, ensureObjectPayload, ensureNonEmptyString } = require('./helpers'); const quoteIdentifier = identifier => `\`${String(identifier ?? '').replace(/`/g, '``')}\``; const normalizeSelectFields = (selectFields, fieldParser) => { if (!Array.isArray(selectFields) || !selectFields.length) { throw 'missing param: selectFields'; } const uniq = new Set(); const normalized = []; for (const item of selectFields) { const field = fieldParser(item, 'selectFields'); if (uniq.has(field)) continue; uniq.add(field); normalized.push(field); } if (!normalized.length) throw 'missing param: selectFields'; return normalized; }; const normalizeGroupBy = (groupBy, fieldParser) => { if (groupBy === undefined || groupBy === null) return []; if (!Array.isArray(groupBy)) throw 'invalid param: groupBy'; const uniq = new Set(); const normalized = []; for (const item of groupBy) { const field = fieldParser(item, 'groupBy'); if (uniq.has(field)) continue; uniq.add(field); normalized.push(field); } return normalized; }; const normalizeOrderBy = (orderBy, fieldParser, label = 'orderBy') => { if (orderBy === undefined || orderBy === null) return []; if (!Array.isArray(orderBy)) throw `invalid param: ${label}`; return orderBy.map((item, index) => { const row = ensureObjectPayload(item, `${label}[${index}]`); const field = fieldParser(row.field, `${label}[${index}].field`); const direction = String(row.direction || 'ASC').trim().toUpperCase(); if (!CLICKHOUSE_SORT_DIRECTIONS.has(direction)) { throw `invalid param: ${label}[${index}].direction`; } return { field, direction }; }); }; const normalizeAggregations = (aggregations, fieldParser, aliasParser) => { if (aggregations === undefined || aggregations === null) return []; if (!Array.isArray(aggregations)) throw 'invalid param: aggregations'; const aliasSet = new Set(); return aggregations.map((item, index) => { const row = ensureObjectPayload(item, `aggregations[${index}]`); const funcRaw = String(row.func || '').trim(); const funcKey = funcRaw.toLowerCase(); const func = CLICKHOUSE_AGGREGATE_FUNCTION_MAP[funcKey]; if (!func) { throw `invalid param: aggregations[${index}].func`; } const field = fieldParser(row.field, `aggregations[${index}].field`); const aliasRaw = hasOwn(row, 'alias') ? row.alias : `${funcKey}_${field}`; const alias = aliasParser(aliasRaw, `aggregations[${index}].alias`); if (aliasSet.has(alias)) throw `invalid param: duplicate aggregation alias (${alias})`; aliasSet.add(alias); return { func, field, alias, }; }); }; const normalizeFilters = (filters, fieldParser) => { if (filters === undefined || filters === null) return []; if (!Array.isArray(filters)) throw 'invalid param: filters'; return filters.map((item, index) => { const row = ensureObjectPayload(item, `filters[${index}]`); const field = fieldParser(row.field, `filters[${index}].field`); const operator = String(row.operator || '').trim().toUpperCase(); if (!CLICKHOUSE_FILTER_OPERATORS.has(operator)) { throw `invalid param: filters[${index}].operator`; } if (!hasOwn(row, 'value')) throw `missing param: filters[${index}].value`; if (operator === 'IN') { if (!Array.isArray(row.value) || !row.value.length) throw `invalid param: filters[${index}].value`; } else if (operator === 'BETWEEN') { if (!Array.isArray(row.value) || row.value.length !== 2) throw `invalid param: filters[${index}].value`; } else if (operator === 'LIKE') { if (!String(row.value || '').trim()) throw `invalid param: filters[${index}].value`; } return { field, operator, value: row.value, }; }); }; const normalizeTimeRange = (timeRange, fieldParser) => { if (timeRange === undefined || timeRange === null) return null; const safeRange = ensureObjectPayload(timeRange, 'timeRange'); const field = fieldParser(safeRange.field, 'timeRange.field'); const start = ensureNonEmptyString(safeRange.start, 'timeRange.start'); const end = ensureNonEmptyString(safeRange.end, 'timeRange.end'); return { field, start, end }; }; const normalizeLimit = rawLimit => { if (rawLimit === undefined || rawLimit === null || rawLimit === '') return 100000; const parsed = toInt(rawLimit); if (!parsed || parsed < 1) throw 'invalid param: limit'; return Math.min(parsed, 100000); }; const normalizeStructuredQueryDsl = (rawDsl = {}, options = {}) => { const tableParser = options.tableParser || ensureNonEmptyString; const fieldParser = options.fieldParser || ensureNonEmptyString; const aliasParser = options.aliasParser || fieldParser; const safeDsl = ensureObjectPayload(rawDsl, 'query'); const table = tableParser(safeDsl.table, 'table'); const selectFields = normalizeSelectFields(safeDsl.selectFields, fieldParser); const aggregations = normalizeAggregations(safeDsl.aggregations, fieldParser, aliasParser); const filters = normalizeFilters(safeDsl.filters, fieldParser); const timeRange = normalizeTimeRange(safeDsl.timeRange, fieldParser); const groupBy = normalizeGroupBy(safeDsl.groupBy, fieldParser); const orderBy = normalizeOrderBy(safeDsl.orderBy, fieldParser); const sourceOrderBy = normalizeOrderBy(safeDsl.sourceOrderBy, fieldParser, 'sourceOrderBy'); const limit = normalizeLimit(safeDsl.limit); return { table, selectFields, aggregations, filters, timeRange, groupBy, orderBy, sourceOrderBy, limit, }; }; const assertFieldsInFieldSet = (fields, fieldSet, label) => { for (const field of fields) { if (!fieldSet.has(field)) throw `invalid param: ${label} (${field})`; } }; const validateNormalizedStructuredQueryDsl = (normalized, fieldSet) => { assertFieldsInFieldSet(normalized.selectFields, fieldSet, 'selectFields'); assertFieldsInFieldSet(normalized.aggregations.map(item => item.field), fieldSet, 'aggregations.field'); assertFieldsInFieldSet(normalized.groupBy, fieldSet, 'groupBy'); assertFieldsInFieldSet(normalized.sourceOrderBy.map(item => item.field), fieldSet, 'sourceOrderBy'); assertFieldsInFieldSet(normalized.filters.map(item => item.field), fieldSet, 'filters'); if (normalized.timeRange) { assertFieldsInFieldSet([normalized.timeRange.field], fieldSet, 'timeRange.field'); } const aggregationAliasSet = new Set(normalized.aggregations.map(item => item.alias)); if (normalized.groupBy.length) { const invalidOrderByFields = normalized.orderBy .map(item => item.field) .filter(field => !normalized.groupBy.includes(field) && !aggregationAliasSet.has(field)); if (invalidOrderByFields.length) { throw 'in groupBy mode, all orderBy fields must be included in groupBy or aggregation aliases'; } } else { assertFieldsInFieldSet(normalized.orderBy.map(item => item.field), fieldSet, 'orderBy'); } if (normalized.groupBy.length) { const notInGroup = normalized.selectFields.filter(field => !normalized.groupBy.includes(field)); if (notInGroup.length) { throw 'in groupBy mode, all selectFields must be included in groupBy'; } } if (normalized.aggregations.length && !normalized.groupBy.length) { throw 'aggregations requires groupBy'; } if (normalized.sourceOrderBy.length && (!normalized.groupBy.length || !normalized.aggregations.length)) { throw 'sourceOrderBy requires groupBy with aggregations'; } return normalized; }; const validateStructuredQueryDslAgainstFields = (rawDsl, fieldSet, options = {}) => { const normalized = normalizeStructuredQueryDsl(rawDsl, options); return validateNormalizedStructuredQueryDsl(normalized, fieldSet); }; const getClickHouseParamType = value => { if (typeof value === 'boolean') return 'UInt8'; if (typeof value === 'number' && Number.isFinite(value)) { return Number.isInteger(value) ? 'Int64' : 'Float64'; } return 'String'; }; const getClickHouseParamValue = value => { if (typeof value === 'boolean') return value ? 1 : 0; if (typeof value === 'number' && Number.isFinite(value)) return value; return String(value ?? ''); }; const buildQueryParamPlaceholder = (state, value) => { state.index += 1; const key = `p${state.index}`; state.queryParams[key] = getClickHouseParamValue(value); return `{${key}:${getClickHouseParamType(value)}}`; }; const buildWhereClause = (dsl, paramState) => { const whereParts = []; for (const filter of dsl.filters) { const fieldSql = quoteIdentifier(filter.field); if ((filter.operator === '=' || filter.operator === '!=') && filter.value === null) { whereParts.push(filter.operator === '=' ? `${fieldSql} IS NULL` : `${fieldSql} IS NOT NULL`); continue; } if (filter.operator === 'IN') { const values = filter.value || []; const placeholders = values.map(value => buildQueryParamPlaceholder(paramState, value)); whereParts.push(`${fieldSql} IN (${placeholders.join(', ')})`); continue; } if (filter.operator === 'BETWEEN') { const left = buildQueryParamPlaceholder(paramState, filter.value[0]); const right = buildQueryParamPlaceholder(paramState, filter.value[1]); whereParts.push(`${fieldSql} BETWEEN ${left} AND ${right}`); continue; } const placeholder = buildQueryParamPlaceholder(paramState, filter.value); whereParts.push(`${fieldSql} ${filter.operator} ${placeholder}`); } if (dsl.timeRange) { const fieldSql = quoteIdentifier(dsl.timeRange.field); const left = buildQueryParamPlaceholder(paramState, dsl.timeRange.start); const right = buildQueryParamPlaceholder(paramState, dsl.timeRange.end); whereParts.push(`${fieldSql} BETWEEN ${left} AND ${right}`); } return whereParts.length ? ` WHERE ${whereParts.join(' AND ')}` : ''; }; const uniqueFields = (fields = []) => { const uniq = new Set(); const normalized = []; for (const field of fields) { if (!field || uniq.has(field)) continue; uniq.add(field); normalized.push(field); } return normalized; }; const buildOrderByClause = orderBy => { if (!orderBy.length) return ''; return ` ORDER BY ${orderBy.map(item => `${quoteIdentifier(item.field)} ${item.direction}`).join(', ')}`; }; const buildAggregationSelectSql = (aggregation, dslLimit) => { const fieldSql = quoteIdentifier(aggregation.field); const aliasSql = quoteIdentifier(aggregation.alias); if (aggregation.func === 'groupArray') { return `${aggregation.func}(${dslLimit})(${fieldSql}) AS ${aliasSql}`; } return `${aggregation.func}(${fieldSql}) AS ${aliasSql}`; }; const buildStructuredResultColumns = dsl => { return [ ...dsl.selectFields, ...dsl.aggregations.map(item => item.alias), ]; }; const buildStructuredPreviewSql = (dsl, options = {}) => { const paramState = { index: 0, queryParams: {} }; const selectParts = [ ...dsl.selectFields.map(quoteIdentifier), ...dsl.aggregations.map(item => buildAggregationSelectSql(item, dsl.limit)), ]; const selectClause = selectParts.join(', '); const tableSql = String(options.tableSql || quoteIdentifier(dsl.table)).trim(); const whereClause = buildWhereClause(dsl, paramState); const sourceOrderByClause = buildOrderByClause(dsl.sourceOrderBy); const fromClause = dsl.sourceOrderBy.length ? ` FROM (SELECT ${ uniqueFields([ ...dsl.selectFields, ...dsl.groupBy, ...dsl.aggregations.map(item => item.field), ...dsl.sourceOrderBy.map(item => item.field), ]).map(quoteIdentifier).join(', ') } FROM ${tableSql}${whereClause}${sourceOrderByClause})` : ` FROM ${tableSql}${whereClause}`; const groupByClause = dsl.groupBy.length ? ` GROUP BY ${dsl.groupBy.map(quoteIdentifier).join(', ')}` : ''; const orderByClause = buildOrderByClause(dsl.orderBy); const limitClause = ` LIMIT ${dsl.limit}`; const sqlPreview = `SELECT ${selectClause}${fromClause}${groupByClause}${orderByClause}${limitClause}`; return { sqlPreview, query: `${sqlPreview} FORMAT JSON`, tableSql, whereClause, hasGroupBy: Boolean(dsl.groupBy.length), resultColumns: buildStructuredResultColumns(dsl), queryParams: paramState.queryParams, }; }; module.exports = { quoteIdentifier, normalizeStructuredQueryDsl, validateNormalizedStructuredQueryDsl, validateStructuredQueryDslAgainstFields, buildStructuredPreviewSql, };