fix(query-generator): generate subQuery filter for nested required joins (#9188)

lib/dialects/abstract/query-generator.js

@@ -1249,7 +1249,7 @@ const QueryGenerator = {
           ? this.quoteIdentifiers(attr)
           ? this.quoteIdentifiers(attr)
           : this.escape(attr);
           : this.escape(attr);
       }
       }
-      if (options.include && attr.indexOf('.') === -1 && addTable) {
+      if (!_.isEmpty(options.include) && attr.indexOf('.') === -1 && addTable) {
         attr = mainTableAs + '.' + attr;
         attr = mainTableAs + '.' + attr;
       }
       }
 
 
@@ -1258,7 +1258,6 @@ const QueryGenerator = {
   },
   },
 
 
   generateInclude(include, parentTableName, topLevelInfo) {
   generateInclude(include, parentTableName, topLevelInfo) {
-    const association = include.association;
     const joinQueries = {
     const joinQueries = {
       mainQuery: [],
       mainQuery: [],
       subQuery: []
       subQuery: []
@@ -1334,43 +1333,7 @@ const QueryGenerator = {
     if (include.through) {
     if (include.through) {
       joinQuery = this.generateThroughJoin(include, includeAs, parentTableName.internalAs, topLevelInfo);
       joinQuery = this.generateThroughJoin(include, includeAs, parentTableName.internalAs, topLevelInfo);
     } else {
     } else {
-      if (topLevelInfo.subQuery && include.subQueryFilter) {
-        const associationWhere = {};
-
-        associationWhere[association.identifierField] = {
-          [Op.eq]: this.sequelize.literal(`${this.quoteTable(parentTableName.internalAs)}.${this.quoteIdentifier(association.sourceKeyField || association.source.primaryKeyField)}`)
-        };
-
-        if (!topLevelInfo.options.where) {
-          topLevelInfo.options.where = {};
-        }
-
-        // Creating the as-is where for the subQuery, checks that the required association exists
-        const $query = this.selectQuery(include.model.getTableName(), {
-          attributes: [association.identifierField],
-          where: {
-            [Op.and]: [
-              associationWhere,
-              include.where || {}
-            ]
-          },
-          limit: 1,
-          tableAs: include.as
-        }, include.model);
-
-        const subQueryWhere = this.sequelize.asIs([
-          '(',
-          $query.replace(/\;$/, ''),
-          ')',
-          'IS NOT NULL'
-        ].join(' '));
-
-        if (_.isPlainObject(topLevelInfo.options.where)) {
-          topLevelInfo.options.where['__' + includeAs.internalAs] = subQueryWhere;
-        } else {
-          topLevelInfo.options.where = { [Op.and]: [topLevelInfo.options.where, subQueryWhere] };
-        }
-      }
+      this._generateSubQueryFilter(include, includeAs, topLevelInfo);
       joinQuery = this.generateJoin(include, topLevelInfo);
       joinQuery = this.generateJoin(include, topLevelInfo);
     }
     }
 
 
@@ -1609,22 +1572,56 @@ const QueryGenerator = {
           joinCondition += ` AND ${targetWhere}`;
           joinCondition += ` AND ${targetWhere}`;
         }
         }
       }
       }
-      if (topLevelInfo.subQuery && include.required) {
+    }
+
+    this._generateSubQueryFilter(include, includeAs, topLevelInfo);
+
+    return {
+      join: joinType,
+      body: joinBody,
+      condition: joinCondition,
+      attributes
+    };
+  },
+
+  /*
+   * Generates subQueryFilter - a select nested in the where clause of the subQuery.
+   * For a given include a query is generated that contains all the way from the subQuery
+   * table to the include table plus everything that's in required transitive closure of the
+   * given include.
+   */
+  _generateSubQueryFilter(include, includeAs, topLevelInfo) {
+    if (!topLevelInfo.subQuery || !include.subQueryFilter) {
+      return;
+    }
+
     if (!topLevelInfo.options.where) {
     if (!topLevelInfo.options.where) {
       topLevelInfo.options.where = {};
       topLevelInfo.options.where = {};
     }
     }
     let parent = include;
     let parent = include;
     let child = include;
     let child = include;
-        let nestedIncludes = [];
+    let nestedIncludes = this._getRequiredClosure(include).include;
     let query;
     let query;
 
 
     while ((parent = parent.parent)) { // eslint-disable-line
     while ((parent = parent.parent)) { // eslint-disable-line
-          nestedIncludes = [_.extend({}, child, { include: nestedIncludes })];
+      if (parent.parent && !parent.required) {
+        return; // only generate subQueryFilter if all the parents of this include are required
+      }
+
+      if (parent.subQueryFilter) {
+        // the include is already handled as this parent has the include on its required closure
+        // skip to prevent duplicate subQueryFilter
+        return;
+      }
+
+      nestedIncludes = [_.extend({}, child, { include: nestedIncludes, attributes: [] })];
       child = parent;
       child = parent;
     }
     }
 
 
     const topInclude = nestedIncludes[0];
     const topInclude = nestedIncludes[0];
     const topParent = topInclude.parent;
     const topParent = topInclude.parent;
+    const topAssociation = topInclude.association;
+    topInclude.association = undefined;
 
 
     if (topInclude.through && Object(topInclude.through.model) === topInclude.through.model) {
     if (topInclude.through && Object(topInclude.through.model) === topInclude.through.model) {
       query = this.selectQuery(topInclude.through.model.getTableName(), {
       query = this.selectQuery(topInclude.through.model.getTableName(), {
@@ -1632,8 +1629,10 @@ const QueryGenerator = {
         include: Model._validateIncludedElements({
         include: Model._validateIncludedElements({
           model: topInclude.through.model,
           model: topInclude.through.model,
           include: [{
           include: [{
-                association: topInclude.association.toTarget,
-                required: true
+            association: topAssociation.toTarget,
+            required: true,
+            where: topInclude.where,
+            include: topInclude.include
           }]
           }]
         }).include,
         }).include,
         model: topInclude.through.model,
         model: topInclude.through.model,
@@ -1641,7 +1640,7 @@ const QueryGenerator = {
           [Op.and]: [
           [Op.and]: [
             this.sequelize.asIs([
             this.sequelize.asIs([
               this.quoteTable(topParent.model.name) + '.' + this.quoteIdentifier(topParent.model.primaryKeyField),
               this.quoteTable(topParent.model.name) + '.' + this.quoteIdentifier(topParent.model.primaryKeyField),
-                  this.quoteIdentifier(topInclude.through.model.name) + '.' + this.quoteIdentifier(topInclude.association.identifierField)
+              this.quoteIdentifier(topInclude.through.model.name) + '.' + this.quoteIdentifier(topAssociation.identifierField)
             ].join(' = ')),
             ].join(' = ')),
             topInclude.through.where
             topInclude.through.where
           ]
           ]
@@ -1650,36 +1649,56 @@ const QueryGenerator = {
         includeIgnoreAttributes: false
         includeIgnoreAttributes: false
       }, topInclude.through.model);
       }, topInclude.through.model);
     } else {
     } else {
-          const isBelongsTo = topInclude.association.associationType === 'BelongsTo';
+      const isBelongsTo = topAssociation.associationType === 'BelongsTo';
+      const sourceField = isBelongsTo ? topAssociation.identifierField : (topAssociation.sourceKeyField || topParent.model.primaryKeyField);
+      const targetField = isBelongsTo ? (topAssociation.sourceKeyField || topInclude.model.primaryKeyField) : topAssociation.identifierField;
+
       const join = [
       const join = [
-            this.quoteTable(topParent.model.name) + '.' + this.quoteIdentifier(isBelongsTo ? topInclude.association.identifierField : topParent.model.primaryKeyAttributes[0]),
-            this.quoteIdentifier(topInclude.model.name) + '.' + this.quoteIdentifier(isBelongsTo ? topInclude.model.primaryKeyAttributes[0] : topInclude.association.identifierField)
+        this.quoteIdentifier(topInclude.as) + '.' + this.quoteIdentifier(targetField),
+        this.quoteTable(topParent.as || topParent.model.name) + '.' + this.quoteIdentifier(sourceField)
       ].join(' = ');
       ].join(' = ');
-          query = this.selectQuery(topInclude.model.tableName, {
-            attributes: [topInclude.model.primaryKeyAttributes[0]],
-            include: topInclude.include,
+
+      query = this.selectQuery(topInclude.model.getTableName(), {
+        attributes: [targetField],
+        include: Model._validateIncludedElements(topInclude).include,
+        model: topInclude.model,
         where: {
         where: {
+          [Op.and]: [{
             [Op.join]: this.sequelize.asIs(join)
             [Op.join]: this.sequelize.asIs(join)
+          }]
         },
         },
         limit: 1,
         limit: 1,
+        tableAs: topInclude.as,
         includeIgnoreAttributes: false
         includeIgnoreAttributes: false
       }, topInclude.model);
       }, topInclude.model);
     }
     }
-        topLevelInfo.options.where['__' + throughAs] = this.sequelize.asIs([
+
+    if (!topLevelInfo.options.where[Op.and]) {
+      topLevelInfo.options.where[Op.and] = [];
+    }
+
+    topLevelInfo.options.where[`__${includeAs.internalAs}`] = this.sequelize.asIs([
       '(',
       '(',
       query.replace(/\;$/, ''),
       query.replace(/\;$/, ''),
       ')',
       ')',
       'IS NOT NULL'
       'IS NOT NULL'
     ].join(' '));
     ].join(' '));
-      }
+  },
+
+  /*
+   * For a given include hierarchy creates a copy of it where only the required includes
+   * are preserved.
+   */
+  _getRequiredClosure(include) {
+    const copy = _.extend({}, include, {attributes: [], include: []});
+
+    if (Array.isArray(include.include)) {
+      copy.include = include.include
+        .filter(i => i.required)
+        .map(inc => this._getRequiredClosure(inc));
     }
     }
 
 
-    return {
-      join: joinType,
-      body: joinBody,
-      condition: joinCondition,
-      attributes
-    };
+    return copy;
   },
   },
 
 
   getQueryOrders(options, model, subQuery) {
   getQueryOrders(options, model, subQuery) {

lib/model.js

@@ -390,6 +390,7 @@ class Model {
     options.include = options.include.map(include => {
     options.include = options.include.map(include => {
       include = this._conformInclude(include);
       include = this._conformInclude(include);
       include.parent = options;
       include.parent = options;
+      include.topLimit = options.topLimit;
 
 
       this._validateIncludedElement.call(options.model, include, tableNames, options);
       this._validateIncludedElement.call(options.model, include, tableNames, options);