fix(query-generator): generate subQuery filter for nested required joins (#9188)

lib/dialects/abstract/query-generator.js

@@ -1249,7 +1249,7 @@ const QueryGenerator = {
           ? this.quoteIdentifiers(attr)
           ? this.quoteIdentifiers(attr)
           : this.escape(attr);
           : this.escape(attr);
       }
       }
-      if (options.include && attr.indexOf('.') === -1 && addTable) {
+      if (!_.isEmpty(options.include) && attr.indexOf('.') === -1 && addTable) {
         attr = mainTableAs + '.' + attr;
         attr = mainTableAs + '.' + attr;
       }
       }
 
 
@@ -1258,7 +1258,6 @@ const QueryGenerator = {
   },
   },
 
 
   generateInclude(include, parentTableName, topLevelInfo) {
   generateInclude(include, parentTableName, topLevelInfo) {
-    const association = include.association;
     const joinQueries = {
     const joinQueries = {
       mainQuery: [],
       mainQuery: [],
       subQuery: []
       subQuery: []
@@ -1334,43 +1333,7 @@ const QueryGenerator = {
     if (include.through) {
     if (include.through) {
       joinQuery = this.generateThroughJoin(include, includeAs, parentTableName.internalAs, topLevelInfo);
       joinQuery = this.generateThroughJoin(include, includeAs, parentTableName.internalAs, topLevelInfo);
     } else {
     } else {
-      if (topLevelInfo.subQuery && include.subQueryFilter) {
-        const associationWhere = {};
-
-        associationWhere[association.identifierField] = {
-          [Op.eq]: this.sequelize.literal(`${this.quoteTable(parentTableName.internalAs)}.${this.quoteIdentifier(association.sourceKeyField || association.source.primaryKeyField)}`)
-        };
-
-        if (!topLevelInfo.options.where) {
-          topLevelInfo.options.where = {};
-        }
-
-        // Creating the as-is where for the subQuery, checks that the required association exists
-        const $query = this.selectQuery(include.model.getTableName(), {
-          attributes: [association.identifierField],
-          where: {
-            [Op.and]: [
-              associationWhere,
-              include.where || {}
-            ]
-          },
-          limit: 1,
-          tableAs: include.as
-        }, include.model);
-
-        const subQueryWhere = this.sequelize.asIs([
-          '(',
-          $query.replace(/\;$/, ''),
-          ')',
-          'IS NOT NULL'
-        ].join(' '));
-
-        if (_.isPlainObject(topLevelInfo.options.where)) {
-          topLevelInfo.options.where['__' + includeAs.internalAs] = subQueryWhere;
-        } else {
-          topLevelInfo.options.where = { [Op.and]: [topLevelInfo.options.where, subQueryWhere] };
-        }
-      }
+      this._generateSubQueryFilter(include, includeAs, topLevelInfo);
       joinQuery = this.generateJoin(include, topLevelInfo);
       joinQuery = this.generateJoin(include, topLevelInfo);
     }
     }
 
 
@@ -1609,71 +1572,10 @@ const QueryGenerator = {
           joinCondition += ` AND ${targetWhere}`;
           joinCondition += ` AND ${targetWhere}`;
         }
         }
       }
       }
-      if (topLevelInfo.subQuery && include.required) {
-        if (!topLevelInfo.options.where) {
-          topLevelInfo.options.where = {};
-        }
-        let parent = include;
-        let child = include;
-        let nestedIncludes = [];
-        let query;
-
-        while ((parent = parent.parent)) { // eslint-disable-line
-          nestedIncludes = [_.extend({}, child, { include: nestedIncludes })];
-          child = parent;
-        }
-
-        const topInclude = nestedIncludes[0];
-        const topParent = topInclude.parent;
-
-        if (topInclude.through && Object(topInclude.through.model) === topInclude.through.model) {
-          query = this.selectQuery(topInclude.through.model.getTableName(), {
-            attributes: [topInclude.through.model.primaryKeyField],
-            include: Model._validateIncludedElements({
-              model: topInclude.through.model,
-              include: [{
-                association: topInclude.association.toTarget,
-                required: true
-              }]
-            }).include,
-            model: topInclude.through.model,
-            where: {
-              [Op.and]: [
-                this.sequelize.asIs([
-                  this.quoteTable(topParent.model.name) + '.' + this.quoteIdentifier(topParent.model.primaryKeyField),
-                  this.quoteIdentifier(topInclude.through.model.name) + '.' + this.quoteIdentifier(topInclude.association.identifierField)
-                ].join(' = ')),
-                topInclude.through.where
-              ]
-            },
-            limit: 1,
-            includeIgnoreAttributes: false
-          }, topInclude.through.model);
-        } else {
-          const isBelongsTo = topInclude.association.associationType === 'BelongsTo';
-          const join = [
-            this.quoteTable(topParent.model.name) + '.' + this.quoteIdentifier(isBelongsTo ? topInclude.association.identifierField : topParent.model.primaryKeyAttributes[0]),
-            this.quoteIdentifier(topInclude.model.name) + '.' + this.quoteIdentifier(isBelongsTo ? topInclude.model.primaryKeyAttributes[0] : topInclude.association.identifierField)
-          ].join(' = ');
-          query = this.selectQuery(topInclude.model.tableName, {
-            attributes: [topInclude.model.primaryKeyAttributes[0]],
-            include: topInclude.include,
-            where: {
-              [Op.join]: this.sequelize.asIs(join)
-            },
-            limit: 1,
-            includeIgnoreAttributes: false
-          }, topInclude.model);
-        }
-        topLevelInfo.options.where['__' + throughAs] = this.sequelize.asIs([
-          '(',
-          query.replace(/\;$/, ''),
-          ')',
-          'IS NOT NULL'
-        ].join(' '));
-      }
     }
     }
 
 
+    this._generateSubQueryFilter(include, includeAs, topLevelInfo);
+
     return {
     return {
       join: joinType,
       join: joinType,
       body: joinBody,
       body: joinBody,
@@ -1682,6 +1584,123 @@ const QueryGenerator = {
     };
     };
   },
   },
 
 
+  /*
+   * Generates subQueryFilter - a select nested in the where clause of the subQuery.
+   * For a given include a query is generated that contains all the way from the subQuery
+   * table to the include table plus everything that's in required transitive closure of the
+   * given include.
+   */
+  _generateSubQueryFilter(include, includeAs, topLevelInfo) {
+    if (!topLevelInfo.subQuery || !include.subQueryFilter) {
+      return;
+    }
+
+    if (!topLevelInfo.options.where) {
+      topLevelInfo.options.where = {};
+    }
+    let parent = include;
+    let child = include;
+    let nestedIncludes = this._getRequiredClosure(include).include;
+    let query;
+
+    while ((parent = parent.parent)) { // eslint-disable-line
+      if (parent.parent && !parent.required) {
+        return; // only generate subQueryFilter if all the parents of this include are required
+      }
+
+      if (parent.subQueryFilter) {
+        // the include is already handled as this parent has the include on its required closure
+        // skip to prevent duplicate subQueryFilter
+        return;
+      }
+
+      nestedIncludes = [_.extend({}, child, { include: nestedIncludes, attributes: [] })];
+      child = parent;
+    }
+
+    const topInclude = nestedIncludes[0];
+    const topParent = topInclude.parent;
+    const topAssociation = topInclude.association;
+    topInclude.association = undefined;
+
+    if (topInclude.through && Object(topInclude.through.model) === topInclude.through.model) {
+      query = this.selectQuery(topInclude.through.model.getTableName(), {
+        attributes: [topInclude.through.model.primaryKeyField],
+        include: Model._validateIncludedElements({
+          model: topInclude.through.model,
+          include: [{
+            association: topAssociation.toTarget,
+            required: true,
+            where: topInclude.where,
+            include: topInclude.include
+          }]
+        }).include,
+        model: topInclude.through.model,
+        where: {
+          [Op.and]: [
+            this.sequelize.asIs([
+              this.quoteTable(topParent.model.name) + '.' + this.quoteIdentifier(topParent.model.primaryKeyField),
+              this.quoteIdentifier(topInclude.through.model.name) + '.' + this.quoteIdentifier(topAssociation.identifierField)
+            ].join(' = ')),
+            topInclude.through.where
+          ]
+        },
+        limit: 1,
+        includeIgnoreAttributes: false
+      }, topInclude.through.model);
+    } else {
+      const isBelongsTo = topAssociation.associationType === 'BelongsTo';
+      const sourceField = isBelongsTo ? topAssociation.identifierField : (topAssociation.sourceKeyField || topParent.model.primaryKeyField);
+      const targetField = isBelongsTo ? (topAssociation.sourceKeyField || topInclude.model.primaryKeyField) : topAssociation.identifierField;
+
+      const join = [
+        this.quoteIdentifier(topInclude.as) + '.' + this.quoteIdentifier(targetField),
+        this.quoteTable(topParent.as || topParent.model.name) + '.' + this.quoteIdentifier(sourceField)
+      ].join(' = ');
+
+      query = this.selectQuery(topInclude.model.getTableName(), {
+        attributes: [targetField],
+        include: Model._validateIncludedElements(topInclude).include,
+        model: topInclude.model,
+        where: {
+          [Op.and]: [{
+            [Op.join]: this.sequelize.asIs(join)
+          }]
+        },
+        limit: 1,
+        tableAs: topInclude.as,
+        includeIgnoreAttributes: false
+      }, topInclude.model);
+    }
+
+    if (!topLevelInfo.options.where[Op.and]) {
+      topLevelInfo.options.where[Op.and] = [];
+    }
+
+    topLevelInfo.options.where[`__${includeAs.internalAs}`] = this.sequelize.asIs([
+      '(',
+      query.replace(/\;$/, ''),
+      ')',
+      'IS NOT NULL'
+    ].join(' '));
+  },
+
+  /*
+   * For a given include hierarchy creates a copy of it where only the required includes
+   * are preserved.
+   */
+  _getRequiredClosure(include) {
+    const copy = _.extend({}, include, {attributes: [], include: []});
+
+    if (Array.isArray(include.include)) {
+      copy.include = include.include
+        .filter(i => i.required)
+        .map(inc => this._getRequiredClosure(inc));
+    }
+
+    return copy;
+  },
+
   getQueryOrders(options, model, subQuery) {
   getQueryOrders(options, model, subQuery) {
     const mainQueryOrder = [];
     const mainQueryOrder = [];
     const subQueryOrder = [];
     const subQueryOrder = [];

lib/model.js

@@ -390,6 +390,7 @@ class Model {
     options.include = options.include.map(include => {
     options.include = options.include.map(include => {
       include = this._conformInclude(include);
       include = this._conformInclude(include);
       include.parent = options;
       include.parent = options;
+      include.topLimit = options.topLimit;
 
 
       this._validateIncludedElement.call(options.model, include, tableNames, options);
       this._validateIncludedElement.call(options.model, include, tableNames, options);