594 lines
23 KiB
JavaScript
594 lines
23 KiB
JavaScript
const _ = require('lodash');
|
|
const debug = require('debug')('mongo-knex:converter');
|
|
const debugExtended = require('debug')('mongo-knex:converter-extended');
|
|
|
|
const logicOps = [
|
|
'$and',
|
|
'$or'
|
|
];
|
|
|
|
const compOps = {
|
|
$eq: '=',
|
|
$ne: '!=',
|
|
$gt: '>',
|
|
$gte: '>=',
|
|
$lt: '<',
|
|
$lte: '<=',
|
|
$in: 'in',
|
|
$nin: 'not in',
|
|
$regex: 'like',
|
|
$not: 'not like'
|
|
};
|
|
|
|
// We don't use a backslash as escpae character, because knex reescapes backslashes in binded parameters
|
|
const likeEscapeCharacter = '*';
|
|
|
|
const isOp = key => key.charAt(0) === '$';
|
|
const isLogicOp = key => isOp(key) && _.includes(logicOps, key);
|
|
const isCompOp = key => isOp(key) && _.includes(_.keys(compOps), key);
|
|
const isNegationOp = key => isOp(key) && _.includes(['$ne', '$nin'], key);
|
|
const isStatementGroupOp = key => _.includes([compOps.$in, compOps.$nin], key);
|
|
|
|
/**
|
|
* JSON Stringify with RegExp support
|
|
* @param {Object} json
|
|
* @returns
|
|
*/
|
|
const stringify = (json) => {
|
|
return JSON.stringify(json, function (key, value) {
|
|
if (value instanceof RegExp) {
|
|
return value.toString();
|
|
}
|
|
return value;
|
|
});
|
|
};
|
|
|
|
const processRegExp = ({source, ignoreCase}) => {
|
|
// A regexp is transformed into a LIKE SQL query.
|
|
// So we need to remove all the regexp escaped characters
|
|
// We don't support any special regexp operators apart from startsWith and endsWith (or both) queries
|
|
source = source.replace(/\\([.*+?^${}()|[\]\\/])/g, '$1');
|
|
|
|
if (ignoreCase) {
|
|
source = source.toLowerCase();
|
|
}
|
|
|
|
// Escape escape character itself
|
|
source = source.replace(new RegExp(_.escapeRegExp(likeEscapeCharacter), 'g'), likeEscapeCharacter + likeEscapeCharacter);
|
|
|
|
// Escape special LIKE characters (% and _)
|
|
source = source.replace(/%/g, likeEscapeCharacter + '%');
|
|
source = source.replace(/_/g, likeEscapeCharacter + '_');
|
|
|
|
// For starts with and ends with in SQL we have to put the wildcard at the opposite end of the string to the regex symbol!
|
|
if (source.startsWith('^')) {
|
|
source = source.substring(1) + '%';
|
|
} else if (source.endsWith('$')) {
|
|
source = '%' + source.substring(0, source.length - 1);
|
|
} else {
|
|
source = '%' + source + '%';
|
|
}
|
|
|
|
return {source, ignoreCase};
|
|
};
|
|
|
|
class MongoToKnex {
|
|
/**
|
|
*
|
|
* @param {Object} options
|
|
* @param {String} options.tableName
|
|
*
|
|
* @param {Object} config
|
|
* @param {Object} config.relations structure:
|
|
* {[relation-name]}: {
|
|
* tableName: String (e.g. tags)
|
|
* tableNameAs: String (e.g. t, optional)
|
|
* type: String (e.g. manyToMany)
|
|
* joinTable: String (e.g. posts_tags)
|
|
* joinFrom: String (e.g. post_id)
|
|
* joinTo: String (e.g. tag_id)
|
|
* }
|
|
*/
|
|
constructor(options = {}, config = {}) {
|
|
this.tableName = options.tableName;
|
|
this.cte = config.cte;
|
|
this.config = {};
|
|
|
|
Object.assign(this.config, {relations: {}}, config);
|
|
}
|
|
|
|
processWhereType(mode, op, value) {
|
|
if (value === null) {
|
|
return (mode === '$or' ? 'orWhere' : 'where') + (op === '$ne' ? 'NotNull' : 'Null');
|
|
}
|
|
|
|
if (mode === '$or') {
|
|
return 'orWhere';
|
|
}
|
|
|
|
return 'andWhere';
|
|
}
|
|
|
|
/**
|
|
* Determine if statement lives on parent table or if statement refers to a relation.
|
|
*/
|
|
processStatement(column, op, value) {
|
|
const [tableName, columnName] = column.split('.');
|
|
|
|
// CASE: relation?
|
|
if (columnName) {
|
|
debug(tableName, columnName);
|
|
|
|
const table = tableName;
|
|
let relation = this.config.relations[table];
|
|
|
|
if (!relation) {
|
|
// CASE: you want to filter by a column on the join table
|
|
relation = _.find(this.config.relations, (_relation) => {
|
|
return _relation.joinTable === table;
|
|
});
|
|
|
|
// CASE: assume it's a column on the destination table
|
|
if (!relation) {
|
|
return {
|
|
column: column,
|
|
operator: op,
|
|
value: value,
|
|
isRelation: false
|
|
};
|
|
}
|
|
|
|
return {
|
|
joinTable: relation.joinTable,
|
|
table: relation.tableName,
|
|
column: columnName,
|
|
operator: op,
|
|
value: value,
|
|
config: relation,
|
|
isRelation: true
|
|
};
|
|
}
|
|
|
|
return {
|
|
table: tableName,
|
|
column: columnName,
|
|
operator: op,
|
|
value: value,
|
|
config: relation,
|
|
isRelation: true
|
|
};
|
|
}
|
|
|
|
// CASE: fallback, status=draft -> posts.status=draft
|
|
return {
|
|
column: (this.cte && this.cte === true) ? `${column}` : `${this.tableName}.${column}`,
|
|
operator: op,
|
|
value: value,
|
|
isRelation: false
|
|
};
|
|
}
|
|
|
|
/**
|
|
* We group the relations by a unique key.
|
|
* Each grouping will create a sub query.
|
|
*
|
|
* Returns a group structure of following format:
|
|
* {
|
|
* "groupKey": {
|
|
* innerWhereStatements: [],
|
|
* joinFilterStatements: []
|
|
* }
|
|
* }
|
|
*/
|
|
groupRelationStatements(statements, mode) {
|
|
const group = {};
|
|
|
|
// groups depend on the mode of grouping, if its and $and we need to treat a filter on
|
|
// joining table differently than we would with $or
|
|
// e.g. for $or we can create a subquery or group that filter,
|
|
// for $and we have to include joining table filter in every group
|
|
const innerWhereStatements = (mode === '$and')
|
|
? statements.filter(r => !(r.joinTable))
|
|
: statements;
|
|
|
|
_.each(innerWhereStatements, (statement, idx) => {
|
|
/**
|
|
* CASE:
|
|
* - we should not use the same sub query if the column name is the same (two sub queries)
|
|
* - e.g. $and conjunction requires us to use 2 sub queries, because we have to look at each individual tag
|
|
*
|
|
* - we should also not use grouping of negated values for the same reasons as above
|
|
*/
|
|
let createSubGroup = isNegationOp(statement.operator);
|
|
|
|
if (!createSubGroup && group[statement.table]) {
|
|
createSubGroup = _.find(group[statement.table].innerWhereStatements, (innerStatement) => {
|
|
if (innerStatement.column === statement.column) {
|
|
return true;
|
|
}
|
|
});
|
|
}
|
|
|
|
let groupKey = statement.table;
|
|
|
|
if (createSubGroup) {
|
|
groupKey = `${statement.table}_${idx})}`;
|
|
|
|
if (group[groupKey]) {
|
|
//eslint-disable-next-line no-restricted-syntax
|
|
throw new Error('Key collision detected');
|
|
}
|
|
}
|
|
|
|
if (!group[groupKey]) {
|
|
group[groupKey] = {};
|
|
group[groupKey].innerWhereStatements = [];
|
|
}
|
|
|
|
group[groupKey].innerWhereStatements.push(statement);
|
|
});
|
|
|
|
// NOTE: filters applied on join level have to be included when they are
|
|
// a part of $and group
|
|
if (mode === '$and') {
|
|
const joinFilterStatements = statements.filter(r => (r.joinTable));
|
|
|
|
_.each(Object.keys(group), (key) => {
|
|
group[key].joinFilterStatements = joinFilterStatements;
|
|
});
|
|
}
|
|
|
|
return group;
|
|
}
|
|
|
|
/**
|
|
* Build queries for relations.
|
|
*/
|
|
buildRelationQuery(qb, relations, mode) {
|
|
debug(`(buildRelationQuery)`);
|
|
|
|
if (debugExtended.enabled) {
|
|
debugExtended(`(buildRelationQuery) ${stringify(relations)}`);
|
|
}
|
|
|
|
const groupedRelations = this.groupRelationStatements(relations, mode);
|
|
|
|
if (debugExtended.enabled) {
|
|
debugExtended(`(buildRelationQuery) grouped: ${stringify(groupedRelations)}`);
|
|
}
|
|
|
|
// CASE: {tags: [where clause, where clause], tags_123: [where clause], authors: [where clause, where clause]}
|
|
_.each(Object.keys(groupedRelations), (key) => {
|
|
debug(`(buildRelationQuery) build relation for ${key}`);
|
|
|
|
const statements = groupedRelations[key].innerWhereStatements;
|
|
|
|
// CASE: any statement for the same relation should contain the same config
|
|
const reference = statements[0];
|
|
|
|
if (reference.config.type === 'manyToMany') {
|
|
if (_.every(statements.map(s => s.operator), isCompOp)) {
|
|
// CASE: only negate whole group when all the operators in the group are negative,
|
|
// otherwise we cannot combine groups with negated and regular equation operators
|
|
const negateGroup = _.every(statements.map(s => s.operator), (operator) => {
|
|
return isNegationOp(operator);
|
|
});
|
|
|
|
const comp = negateGroup
|
|
? compOps.$nin
|
|
: compOps.$in;
|
|
|
|
const whereType = ['whereNull', 'whereNotNull'].includes(reference.whereType) ? 'andWhere' : (['orWhereNull', 'orWhereNotNull'].includes(reference.whereType) ? 'orWhere' : reference.whereType);
|
|
|
|
// CASE: WHERE resource.id (IN | NOT IN) (SELECT ...)
|
|
qb[whereType](`${this.tableName}.id`, comp, function () {
|
|
const joinFilterStatements = groupedRelations[key].joinFilterStatements;
|
|
|
|
let innerJoinValue = reference.config.tableName;
|
|
let innerJoinOn = `${reference.config.tableName}.${reference.config.joinToForeign || 'id'}`;
|
|
|
|
// CASE: you can define a name for the join table
|
|
if (reference.config.tableNameAs) {
|
|
innerJoinValue = `${reference.config.tableName} as ${reference.config.tableNameAs}`;
|
|
innerJoinOn = `${reference.config.tableNameAs}.${reference.config.joinToForeign || 'id'}`;
|
|
}
|
|
|
|
const joinType = reference.config.joinType || 'innerJoin';
|
|
|
|
const innerQB = this
|
|
.select(`${reference.config.joinTable}.${reference.config.joinFrom}`)
|
|
.from(`${reference.config.joinTable}`)[joinType](innerJoinValue, function () {
|
|
this.on(innerJoinOn, '=', `${reference.config.joinTable}.${reference.config.joinTo}`);
|
|
|
|
// CASE: when applying AND con junction and having multiple groups the filter
|
|
// related to joining table has to be applied within each group
|
|
_.each(joinFilterStatements, (joinFilter) => {
|
|
this.andOn(`${joinFilter.joinTable}.${joinFilter.column}`, compOps[joinFilter.operator], joinFilter.value);
|
|
});
|
|
});
|
|
|
|
if (debugExtended.enabled) {
|
|
debug(`(buildRelationQuery) innerQB sql-pre: ${innerQB.toSQL().sql}`);
|
|
}
|
|
|
|
_.each(statements, (statement, _key) => {
|
|
debug(`(buildRelationQuery) build relation where statements for ${_key}`);
|
|
|
|
const statementColumn = `${statement.joinTable || statement.table}.${statement.column}`;
|
|
let statementOp;
|
|
|
|
if (negateGroup) {
|
|
statementOp = compOps.$in;
|
|
} else {
|
|
if (isNegationOp(statement.operator)) {
|
|
statementOp = compOps.$nin;
|
|
} else {
|
|
statementOp = compOps[statement.operator];
|
|
}
|
|
}
|
|
|
|
let statementValue = statement.value;
|
|
|
|
// CASE: need to normalize value to array when it's a group operation
|
|
if (isStatementGroupOp(statementOp)) {
|
|
statementValue = !_.isArray(statement.value) ? [statement.value] : statement.value;
|
|
}
|
|
|
|
innerQB[statement.whereType](statementColumn, statementOp, statementValue);
|
|
});
|
|
|
|
if (debugExtended.enabled) {
|
|
debug(`(buildRelationQuery) innerQB sql-post: ${innerQB.toSQL().sql}`);
|
|
}
|
|
|
|
return innerQB;
|
|
});
|
|
} else {
|
|
debug(`one of ${key} group statements contains unknown operator`);
|
|
}
|
|
} else if (reference.config.type === 'oneToOne') {
|
|
if (_.every(statements.map(s => s.operator), isCompOp)) {
|
|
// CASE: only negate whole group when all the operators in the group are negative,
|
|
// otherwise we cannot combine groups with negated and regular equation operators
|
|
const negateGroup = _.every(statements.map(s => s.operator), (operator) => {
|
|
return isNegationOp(operator);
|
|
});
|
|
|
|
const comp = negateGroup
|
|
? compOps.$nin
|
|
: compOps.$in;
|
|
const tableName = this.tableName;
|
|
|
|
const where = reference.whereType === 'orWhere' ? 'orWhere' : 'where';
|
|
qb[where](`${this.tableName}.id`, comp, function () {
|
|
const joinFilterStatements = groupedRelations[key].joinFilterStatements;
|
|
|
|
let innerJoinValue = reference.config.tableName;
|
|
let innerJoinOn = `${reference.config.tableName}.${reference.config.joinFrom}`;
|
|
|
|
// CASE: you can define a name for the join table
|
|
if (reference.config.tableNameAs) {
|
|
innerJoinValue = `${reference.config.tableName} as ${reference.config.tableNameAs}`;
|
|
innerJoinOn = `${reference.config.tableNameAs}.${reference.config.joinFrom}`;
|
|
}
|
|
|
|
const innerQB = this
|
|
.select(`${tableName}.id`)
|
|
.from(`${tableName}`)
|
|
.leftJoin(innerJoinValue, function () {
|
|
this.on(innerJoinOn, '=', `${tableName}.id`);
|
|
|
|
// CASE: when applying AND con junction and having multiple groups the filter
|
|
// related to joining table has to be applied within each group
|
|
_.each(joinFilterStatements, (joinFilter) => {
|
|
this.andOn(`${joinFilter.joinTable}.${joinFilter.column}`, compOps[joinFilter.operator], joinFilter.value);
|
|
});
|
|
});
|
|
|
|
_.each(statements, (statement, _key) => {
|
|
debug(`(buildRelationQuery) build relation where statements for ${_key}`);
|
|
|
|
const statementColumn = `${statement.table}.${statement.column}`;
|
|
let statementOp;
|
|
|
|
// NOTE: this negation is here to ensure records with no relation are
|
|
// include in negation (e.g. `relation.columnName: {$ne: null})
|
|
if (negateGroup) {
|
|
statementOp = compOps.$in;
|
|
|
|
if (statement.value === null) {
|
|
statement.whereType = (statement.whereType === 'whereNotNull') ? 'whereNull' : 'whereNotNull';
|
|
}
|
|
} else {
|
|
if (isNegationOp(statement.operator)) {
|
|
statementOp = compOps.$nin;
|
|
} else {
|
|
statementOp = compOps[statement.operator];
|
|
}
|
|
}
|
|
|
|
let statementValue = statement.value;
|
|
|
|
// CASE: need to normalize value to array when it's a group operation
|
|
if (isStatementGroupOp(statementOp)) {
|
|
statementValue = !_.isArray(statement.value) ? [statement.value] : statement.value;
|
|
}
|
|
|
|
innerQB[statement.whereType](statementColumn, statementOp, statementValue);
|
|
});
|
|
|
|
if (debugExtended.enabled) {
|
|
debug(`(buildRelationQuery) innerQB sql-pre: ${innerQB.toSQL().sql}`);
|
|
}
|
|
|
|
return innerQB;
|
|
});
|
|
} else {
|
|
debug(`one of ${key} group statements contains unknown operator`);
|
|
}
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Determines if statement is a simple where comparison on the parent table or if the statement is a relation query.
|
|
*
|
|
* e.g.
|
|
*
|
|
* `where column = value`
|
|
* `where column != value`
|
|
* `where column > value`
|
|
*/
|
|
buildComparison(qb, mode, statement, op, value, group) {
|
|
const comp = compOps[op] || '=';
|
|
const processedStatement = this.processStatement(statement, op, value);
|
|
let whereType = this.processWhereType(mode, op, value);
|
|
|
|
debug(`(buildComparison) mode: ${mode}, op: ${op}, isRelation: ${processedStatement.isRelation}, group: ${group}`);
|
|
|
|
// Call out to build any necessary relation queries
|
|
if (processedStatement.isRelation) {
|
|
processedStatement.whereType = whereType;
|
|
|
|
// CASE: if the statement is not part of a group, execute the query instantly
|
|
if (!group) {
|
|
this.buildRelationQuery(qb, [processedStatement], mode);
|
|
return;
|
|
}
|
|
|
|
// CASE: if the statement is part of a group, collect the relation statements to be able to group them later
|
|
if (!Object.prototype.hasOwnProperty.call(qb, 'relations')) {
|
|
qb.relations = [];
|
|
}
|
|
|
|
qb.relations.push(processedStatement);
|
|
return;
|
|
}
|
|
|
|
// Build the comparisons using our processed data
|
|
const column = processedStatement.column;
|
|
op = processedStatement.operator;
|
|
value = processedStatement.value;
|
|
|
|
if (op === '$regex' || op === '$not') {
|
|
const {source, ignoreCase} = processRegExp(value);
|
|
value = source;
|
|
|
|
// CASE: regex with i flag needs whereRaw to wrap column in lower() else fall through
|
|
if (ignoreCase) {
|
|
whereType += 'Raw';
|
|
debug(`(buildComparison) whereType: ${whereType}, statement: ${statement}, op: ${op}, comp: ${comp}, value: ${value} (REGEX/i)`);
|
|
qb[whereType](`lower(??) ${comp} ? ESCAPE ?`, [column, value, likeEscapeCharacter]);
|
|
return;
|
|
}
|
|
whereType += 'Raw';
|
|
debug(`(buildComparison) whereType: ${whereType}, statement: ${statement}, op: ${op}, comp: ${comp}, value: ${value} (REGEX)`);
|
|
qb[whereType](`?? ${comp} ? ESCAPE ?`, [column, value, likeEscapeCharacter]);
|
|
return;
|
|
}
|
|
|
|
debug(`(buildComparison) whereType: ${whereType}, statement: ${statement}, op: ${op}, comp: ${comp}, value: ${value}`);
|
|
qb[whereType](column, comp, value);
|
|
}
|
|
|
|
/**
|
|
* {author: 'carl'}
|
|
*/
|
|
buildWhereClause(qb, mode, statement, sub, group) {
|
|
debug(`(buildWhereClause) mode: ${mode}, statement: ${statement}`);
|
|
|
|
if (debugExtended.enabled) {
|
|
debugExtended(`(buildWhereClause) ${stringify(sub)}`);
|
|
}
|
|
|
|
// CASE sub is an atomic value, we use "eq" as default operator
|
|
if (!_.isObject(sub)) {
|
|
return this.buildComparison(qb, mode, statement, '$eq', sub, group);
|
|
}
|
|
|
|
// CASE: sub is an object, contains statements and operators
|
|
_.forIn(sub, (value, op) => {
|
|
if (isCompOp(op)) {
|
|
this.buildComparison(qb, mode, statement, op, value, group);
|
|
} else {
|
|
debug('unknown operator');
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* {$and: [{author: 'carl'}, {status: 'draft'}]}}
|
|
* {$and: {author: 'carl'}}
|
|
* {$and: {author: { $in: [...] }}}
|
|
*/
|
|
buildWhereGroup(qb, parentMode, mode, sub) {
|
|
const whereType = this.processWhereType(parentMode);
|
|
|
|
debug(`(buildWhereGroup) mode: ${mode}, whereType: ${whereType}`);
|
|
|
|
if (debugExtended.enabled) {
|
|
debugExtended(`(buildWhereGroup) ${stringify(sub)}`);
|
|
}
|
|
|
|
qb[whereType]((_qb) => {
|
|
if (_.isArray(sub)) {
|
|
sub.forEach(statement => this.buildQuery(_qb, mode, statement, true));
|
|
} else if (_.isObject(sub)) {
|
|
this.buildQuery(_qb, mode, sub, true);
|
|
}
|
|
|
|
// CASE: now execute all relation statements of this group
|
|
if (Object.prototype.hasOwnProperty.call(_qb, 'relations')) {
|
|
this.buildRelationQuery(_qb, _qb.relations, mode);
|
|
delete _qb.relations;
|
|
}
|
|
});
|
|
}
|
|
|
|
buildQuery(qb, mode, sub, group) {
|
|
debug(`(buildQuery) mode: ${mode}`);
|
|
|
|
if (debugExtended.enabled) {
|
|
debugExtended(`(buildQuery) ${stringify(sub)}`);
|
|
}
|
|
|
|
_.forIn(sub, (value, key) => {
|
|
debug(`(buildQuery) key: ${key}`);
|
|
|
|
if (isLogicOp(key)) {
|
|
// CASE: you have two groups ($or), you have one group ($and)
|
|
this.buildWhereGroup(qb, mode, key, value);
|
|
} else {
|
|
this.buildWhereClause(qb, mode, key, value, group);
|
|
}
|
|
});
|
|
}
|
|
|
|
/**
|
|
* The converter receives sub query objects e.g. `qb.where('..', (qb) => {})`, which
|
|
* we then pass around to our class methods. That's why we pass the parent `qb` object
|
|
* around instead of remembering it as `this.qb`. There are multiple `qb` objects.
|
|
*/
|
|
processJSON(qb, mongoJSON) {
|
|
debug('(processJSON)');
|
|
|
|
// DEBUG=mongo-knex:converter,mongo-knex:converter-extended
|
|
if (debugExtended.enabled) {
|
|
debugExtended(`(processJSON) ${stringify(mongoJSON)}`);
|
|
}
|
|
|
|
// 'and' is the default behaviour
|
|
this.buildQuery(qb, '$and', mongoJSON);
|
|
}
|
|
}
|
|
|
|
module.exports = function convertor(qb, mongoJSON, config) {
|
|
const mongoToKnex = new MongoToKnex({
|
|
tableName: qb._single.table
|
|
}, config);
|
|
|
|
mongoToKnex.processJSON(qb, mongoJSON);
|
|
|
|
return qb;
|
|
};
|