You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
324 lines
13 KiB
324 lines
13 KiB
'use strict';
|
|
|
|
const {
|
|
CLICKHOUSE_FILTER_OPERATORS,
|
|
CLICKHOUSE_SORT_DIRECTIONS,
|
|
CLICKHOUSE_AGGREGATE_FUNCTION_MAP,
|
|
} = require('./constants');
|
|
const { hasOwn, toInt, ensureObjectPayload, ensureNonEmptyString } = require('./helpers');
|
|
|
|
const quoteIdentifier = identifier => `\`${String(identifier ?? '').replace(/`/g, '``')}\``;
|
|
|
|
const normalizeSelectFields = (selectFields, fieldParser) => {
|
|
if (!Array.isArray(selectFields) || !selectFields.length) {
|
|
throw 'missing param: selectFields';
|
|
}
|
|
const uniq = new Set();
|
|
const normalized = [];
|
|
for (const item of selectFields) {
|
|
const field = fieldParser(item, 'selectFields');
|
|
if (uniq.has(field)) continue;
|
|
uniq.add(field);
|
|
normalized.push(field);
|
|
}
|
|
if (!normalized.length) throw 'missing param: selectFields';
|
|
return normalized;
|
|
};
|
|
|
|
const normalizeGroupBy = (groupBy, fieldParser) => {
|
|
if (groupBy === undefined || groupBy === null) return [];
|
|
if (!Array.isArray(groupBy)) throw 'invalid param: groupBy';
|
|
const uniq = new Set();
|
|
const normalized = [];
|
|
for (const item of groupBy) {
|
|
const field = fieldParser(item, 'groupBy');
|
|
if (uniq.has(field)) continue;
|
|
uniq.add(field);
|
|
normalized.push(field);
|
|
}
|
|
return normalized;
|
|
};
|
|
|
|
const normalizeOrderBy = (orderBy, fieldParser, label = 'orderBy') => {
|
|
if (orderBy === undefined || orderBy === null) return [];
|
|
if (!Array.isArray(orderBy)) throw `invalid param: ${label}`;
|
|
return orderBy.map((item, index) => {
|
|
const row = ensureObjectPayload(item, `${label}[${index}]`);
|
|
const field = fieldParser(row.field, `${label}[${index}].field`);
|
|
const direction = String(row.direction || 'ASC').trim().toUpperCase();
|
|
if (!CLICKHOUSE_SORT_DIRECTIONS.has(direction)) {
|
|
throw `invalid param: ${label}[${index}].direction`;
|
|
}
|
|
return { field, direction };
|
|
});
|
|
};
|
|
|
|
const normalizeAggregations = (aggregations, fieldParser, aliasParser) => {
|
|
if (aggregations === undefined || aggregations === null) return [];
|
|
if (!Array.isArray(aggregations)) throw 'invalid param: aggregations';
|
|
const aliasSet = new Set();
|
|
return aggregations.map((item, index) => {
|
|
const row = ensureObjectPayload(item, `aggregations[${index}]`);
|
|
const funcRaw = String(row.func || '').trim();
|
|
const funcKey = funcRaw.toLowerCase();
|
|
const func = CLICKHOUSE_AGGREGATE_FUNCTION_MAP[funcKey];
|
|
if (!func) {
|
|
throw `invalid param: aggregations[${index}].func`;
|
|
}
|
|
const field = fieldParser(row.field, `aggregations[${index}].field`);
|
|
const aliasRaw = hasOwn(row, 'alias') ? row.alias : `${funcKey}_${field}`;
|
|
const alias = aliasParser(aliasRaw, `aggregations[${index}].alias`);
|
|
if (aliasSet.has(alias)) throw `invalid param: duplicate aggregation alias (${alias})`;
|
|
aliasSet.add(alias);
|
|
return {
|
|
func,
|
|
field,
|
|
alias,
|
|
};
|
|
});
|
|
};
|
|
|
|
const normalizeFilters = (filters, fieldParser) => {
|
|
if (filters === undefined || filters === null) return [];
|
|
if (!Array.isArray(filters)) throw 'invalid param: filters';
|
|
return filters.map((item, index) => {
|
|
const row = ensureObjectPayload(item, `filters[${index}]`);
|
|
const field = fieldParser(row.field, `filters[${index}].field`);
|
|
const operator = String(row.operator || '').trim().toUpperCase();
|
|
if (!CLICKHOUSE_FILTER_OPERATORS.has(operator)) {
|
|
throw `invalid param: filters[${index}].operator`;
|
|
}
|
|
if (!hasOwn(row, 'value')) throw `missing param: filters[${index}].value`;
|
|
if (operator === 'IN') {
|
|
if (!Array.isArray(row.value) || !row.value.length) throw `invalid param: filters[${index}].value`;
|
|
} else if (operator === 'BETWEEN') {
|
|
if (!Array.isArray(row.value) || row.value.length !== 2) throw `invalid param: filters[${index}].value`;
|
|
} else if (operator === 'LIKE') {
|
|
if (!String(row.value || '').trim()) throw `invalid param: filters[${index}].value`;
|
|
}
|
|
return {
|
|
field,
|
|
operator,
|
|
value: row.value,
|
|
};
|
|
});
|
|
};
|
|
|
|
const normalizeTimeRange = (timeRange, fieldParser) => {
|
|
if (timeRange === undefined || timeRange === null) return null;
|
|
const safeRange = ensureObjectPayload(timeRange, 'timeRange');
|
|
const field = fieldParser(safeRange.field, 'timeRange.field');
|
|
const start = ensureNonEmptyString(safeRange.start, 'timeRange.start');
|
|
const end = ensureNonEmptyString(safeRange.end, 'timeRange.end');
|
|
return { field, start, end };
|
|
};
|
|
|
|
const normalizeLimit = rawLimit => {
|
|
if (rawLimit === undefined || rawLimit === null || rawLimit === '') return 100000;
|
|
const parsed = toInt(rawLimit);
|
|
if (!parsed || parsed < 1) throw 'invalid param: limit';
|
|
return Math.min(parsed, 100000);
|
|
};
|
|
|
|
const normalizeStructuredQueryDsl = (rawDsl = {}, options = {}) => {
|
|
const tableParser = options.tableParser || ensureNonEmptyString;
|
|
const fieldParser = options.fieldParser || ensureNonEmptyString;
|
|
const aliasParser = options.aliasParser || fieldParser;
|
|
const safeDsl = ensureObjectPayload(rawDsl, 'query');
|
|
const table = tableParser(safeDsl.table, 'table');
|
|
const selectFields = normalizeSelectFields(safeDsl.selectFields, fieldParser);
|
|
const aggregations = normalizeAggregations(safeDsl.aggregations, fieldParser, aliasParser);
|
|
const filters = normalizeFilters(safeDsl.filters, fieldParser);
|
|
const timeRange = normalizeTimeRange(safeDsl.timeRange, fieldParser);
|
|
const groupBy = normalizeGroupBy(safeDsl.groupBy, fieldParser);
|
|
const orderBy = normalizeOrderBy(safeDsl.orderBy, fieldParser);
|
|
const sourceOrderBy = normalizeOrderBy(safeDsl.sourceOrderBy, fieldParser, 'sourceOrderBy');
|
|
const limit = normalizeLimit(safeDsl.limit);
|
|
return {
|
|
table,
|
|
selectFields,
|
|
aggregations,
|
|
filters,
|
|
timeRange,
|
|
groupBy,
|
|
orderBy,
|
|
sourceOrderBy,
|
|
limit,
|
|
};
|
|
};
|
|
|
|
const assertFieldsInFieldSet = (fields, fieldSet, label) => {
|
|
for (const field of fields) {
|
|
if (!fieldSet.has(field)) throw `invalid param: ${label} (${field})`;
|
|
}
|
|
};
|
|
|
|
const validateNormalizedStructuredQueryDsl = (normalized, fieldSet) => {
|
|
assertFieldsInFieldSet(normalized.selectFields, fieldSet, 'selectFields');
|
|
assertFieldsInFieldSet(normalized.aggregations.map(item => item.field), fieldSet, 'aggregations.field');
|
|
assertFieldsInFieldSet(normalized.groupBy, fieldSet, 'groupBy');
|
|
assertFieldsInFieldSet(normalized.sourceOrderBy.map(item => item.field), fieldSet, 'sourceOrderBy');
|
|
assertFieldsInFieldSet(normalized.filters.map(item => item.field), fieldSet, 'filters');
|
|
if (normalized.timeRange) {
|
|
assertFieldsInFieldSet([normalized.timeRange.field], fieldSet, 'timeRange.field');
|
|
}
|
|
const aggregationAliasSet = new Set(normalized.aggregations.map(item => item.alias));
|
|
if (normalized.groupBy.length) {
|
|
const invalidOrderByFields = normalized.orderBy
|
|
.map(item => item.field)
|
|
.filter(field => !normalized.groupBy.includes(field) && !aggregationAliasSet.has(field));
|
|
if (invalidOrderByFields.length) {
|
|
throw 'in groupBy mode, all orderBy fields must be included in groupBy or aggregation aliases';
|
|
}
|
|
} else {
|
|
assertFieldsInFieldSet(normalized.orderBy.map(item => item.field), fieldSet, 'orderBy');
|
|
}
|
|
if (normalized.groupBy.length) {
|
|
const notInGroup = normalized.selectFields.filter(field => !normalized.groupBy.includes(field));
|
|
if (notInGroup.length) {
|
|
throw 'in groupBy mode, all selectFields must be included in groupBy';
|
|
}
|
|
}
|
|
if (normalized.aggregations.length && !normalized.groupBy.length) {
|
|
throw 'aggregations requires groupBy';
|
|
}
|
|
if (normalized.sourceOrderBy.length && (!normalized.groupBy.length || !normalized.aggregations.length)) {
|
|
throw 'sourceOrderBy requires groupBy with aggregations';
|
|
}
|
|
return normalized;
|
|
};
|
|
|
|
const validateStructuredQueryDslAgainstFields = (rawDsl, fieldSet, options = {}) => {
|
|
const normalized = normalizeStructuredQueryDsl(rawDsl, options);
|
|
return validateNormalizedStructuredQueryDsl(normalized, fieldSet);
|
|
};
|
|
|
|
const getClickHouseParamType = value => {
|
|
if (typeof value === 'boolean') return 'UInt8';
|
|
if (typeof value === 'number' && Number.isFinite(value)) {
|
|
return Number.isInteger(value) ? 'Int64' : 'Float64';
|
|
}
|
|
return 'String';
|
|
};
|
|
|
|
const getClickHouseParamValue = value => {
|
|
if (typeof value === 'boolean') return value ? 1 : 0;
|
|
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
|
return String(value ?? '');
|
|
};
|
|
|
|
const buildQueryParamPlaceholder = (state, value) => {
|
|
state.index += 1;
|
|
const key = `p${state.index}`;
|
|
state.queryParams[key] = getClickHouseParamValue(value);
|
|
return `{${key}:${getClickHouseParamType(value)}}`;
|
|
};
|
|
|
|
const buildWhereClause = (dsl, paramState) => {
|
|
const whereParts = [];
|
|
for (const filter of dsl.filters) {
|
|
const fieldSql = quoteIdentifier(filter.field);
|
|
if ((filter.operator === '=' || filter.operator === '!=') && filter.value === null) {
|
|
whereParts.push(filter.operator === '=' ? `${fieldSql} IS NULL` : `${fieldSql} IS NOT NULL`);
|
|
continue;
|
|
}
|
|
if (filter.operator === 'IN') {
|
|
const values = filter.value || [];
|
|
const placeholders = values.map(value => buildQueryParamPlaceholder(paramState, value));
|
|
whereParts.push(`${fieldSql} IN (${placeholders.join(', ')})`);
|
|
continue;
|
|
}
|
|
if (filter.operator === 'BETWEEN') {
|
|
const left = buildQueryParamPlaceholder(paramState, filter.value[0]);
|
|
const right = buildQueryParamPlaceholder(paramState, filter.value[1]);
|
|
whereParts.push(`${fieldSql} BETWEEN ${left} AND ${right}`);
|
|
continue;
|
|
}
|
|
const placeholder = buildQueryParamPlaceholder(paramState, filter.value);
|
|
whereParts.push(`${fieldSql} ${filter.operator} ${placeholder}`);
|
|
}
|
|
if (dsl.timeRange) {
|
|
const fieldSql = quoteIdentifier(dsl.timeRange.field);
|
|
const left = buildQueryParamPlaceholder(paramState, dsl.timeRange.start);
|
|
const right = buildQueryParamPlaceholder(paramState, dsl.timeRange.end);
|
|
whereParts.push(`${fieldSql} BETWEEN ${left} AND ${right}`);
|
|
}
|
|
return whereParts.length ? ` WHERE ${whereParts.join(' AND ')}` : '';
|
|
};
|
|
|
|
const uniqueFields = (fields = []) => {
|
|
const uniq = new Set();
|
|
const normalized = [];
|
|
for (const field of fields) {
|
|
if (!field || uniq.has(field)) continue;
|
|
uniq.add(field);
|
|
normalized.push(field);
|
|
}
|
|
return normalized;
|
|
};
|
|
|
|
const buildOrderByClause = orderBy => {
|
|
if (!orderBy.length) return '';
|
|
return ` ORDER BY ${orderBy.map(item => `${quoteIdentifier(item.field)} ${item.direction}`).join(', ')}`;
|
|
};
|
|
|
|
const buildAggregationSelectSql = (aggregation, dslLimit) => {
|
|
const fieldSql = quoteIdentifier(aggregation.field);
|
|
const aliasSql = quoteIdentifier(aggregation.alias);
|
|
if (aggregation.func === 'groupArray') {
|
|
return `${aggregation.func}(${dslLimit})(${fieldSql}) AS ${aliasSql}`;
|
|
}
|
|
return `${aggregation.func}(${fieldSql}) AS ${aliasSql}`;
|
|
};
|
|
|
|
const buildStructuredResultColumns = dsl => {
|
|
return [
|
|
...dsl.selectFields,
|
|
...dsl.aggregations.map(item => item.alias),
|
|
];
|
|
};
|
|
|
|
const buildStructuredPreviewSql = (dsl, options = {}) => {
|
|
const paramState = { index: 0, queryParams: {} };
|
|
const selectParts = [
|
|
...dsl.selectFields.map(quoteIdentifier),
|
|
...dsl.aggregations.map(item => buildAggregationSelectSql(item, dsl.limit)),
|
|
];
|
|
const selectClause = selectParts.join(', ');
|
|
const tableSql = String(options.tableSql || quoteIdentifier(dsl.table)).trim();
|
|
const whereClause = buildWhereClause(dsl, paramState);
|
|
const sourceOrderByClause = buildOrderByClause(dsl.sourceOrderBy);
|
|
const fromClause = dsl.sourceOrderBy.length
|
|
? ` FROM (SELECT ${
|
|
uniqueFields([
|
|
...dsl.selectFields,
|
|
...dsl.groupBy,
|
|
...dsl.aggregations.map(item => item.field),
|
|
...dsl.sourceOrderBy.map(item => item.field),
|
|
]).map(quoteIdentifier).join(', ')
|
|
} FROM ${tableSql}${whereClause}${sourceOrderByClause})`
|
|
: ` FROM ${tableSql}${whereClause}`;
|
|
const groupByClause = dsl.groupBy.length
|
|
? ` GROUP BY ${dsl.groupBy.map(quoteIdentifier).join(', ')}`
|
|
: '';
|
|
const orderByClause = buildOrderByClause(dsl.orderBy);
|
|
const limitClause = ` LIMIT ${dsl.limit}`;
|
|
const sqlPreview = `SELECT ${selectClause}${fromClause}${groupByClause}${orderByClause}${limitClause}`;
|
|
return {
|
|
sqlPreview,
|
|
query: `${sqlPreview} FORMAT JSON`,
|
|
tableSql,
|
|
whereClause,
|
|
hasGroupBy: Boolean(dsl.groupBy.length),
|
|
resultColumns: buildStructuredResultColumns(dsl),
|
|
queryParams: paramState.queryParams,
|
|
};
|
|
};
|
|
|
|
module.exports = {
|
|
quoteIdentifier,
|
|
normalizeStructuredQueryDsl,
|
|
validateNormalizedStructuredQueryDsl,
|
|
validateStructuredQueryDslAgainstFields,
|
|
buildStructuredPreviewSql,
|
|
};
|
|
|