Grouped limit include (#6560)

* add(internals/groupedLimit): Support for single include in grouped limit

* Change to groupedLimit include works

* Allow association in groupedLimit.on. Fix tests

* Let's actually do a proper clone

* Add duplicating: false, remove attributes support

* More tests for groupedLimit. Support for order in btm grouped limit. Query caching in grouped limit

* Removed a stray comment

* Use dot syntax

* Skip groupedLimit test for sqlite

* Add test for groupedLimit with multiple orders

* Yeah, let me just change that assertion back...

* Use the aliased table name for WHERE

lib/associations/has-many.js

@@ -277,7 +277,7 @@ class HasMany extends Association {
       if (options.limit && instances.length > 1) {
       if (options.limit && instances.length > 1) {
         options.groupedLimit = {
         options.groupedLimit = {
           limit: options.limit,
           limit: options.limit,
-          on: association.foreignKeyField,
+          on: association,
           values
           values
         };
         };
 
 

lib/dialects/abstract/query-generator.js

@@ -8,6 +8,8 @@ const _ = require('lodash');
 const util = require('util');
 const util = require('util');
 const Dottie = require('dottie');
 const Dottie = require('dottie');
 const BelongsTo = require('../../associations/belongs-to');
 const BelongsTo = require('../../associations/belongs-to');
+const BelongsToMany = require('../../associations/belongs-to-many');
+const HasMany = require('../../associations/has-many');
 const uuid = require('node-uuid');
 const uuid = require('node-uuid');
 const semver = require('semver');
 const semver = require('semver');
 
 
@@ -1384,21 +1386,101 @@ const QueryGenerator = {
         if (!mainTableAs) {
         if (!mainTableAs) {
           mainTableAs = table;
           mainTableAs = table;
         }
         }
+
+        const where = Object.assign({}, options.where);
+        let groupedLimitOrder
+          , whereKey
+          , include
+          , groupedTableName = mainTableAs;
+
+        if (typeof options.groupedLimit.on === 'string') {
+          whereKey = options.groupedLimit.on;
+        } else if (options.groupedLimit.on instanceof HasMany) {
+          whereKey = options.groupedLimit.on.foreignKeyField;
+        }
+
+        if (options.groupedLimit.on instanceof BelongsToMany) {
+          // BTM includes needs to join the through table on to check ID
+          groupedTableName = options.groupedLimit.on.manyFromSource.as;
+          const groupedLimitOptions = Model._validateIncludedElements({
+            include: [{
+              association: options.groupedLimit.on.manyFromSource,
+              duplicating: false, // The UNION'ed query may contain duplicates, but each sub-query cannot
+              required: true,
+              where: {
+                '$$PLACEHOLDER$$': true
+              }
+            }],
+            model
+          });
+
+          // Make sure attributes from the join table are mapped back to models
+          options.hasJoin = true;
+          options.hasMultiAssociation = true;
+          options.includeMap = Object.assign(groupedLimitOptions.includeMap, options.includeMap);
+          options.includeNames = groupedLimitOptions.includeNames.concat(options.includeNames || []);
+          include = groupedLimitOptions.include;
+
+          if (Array.isArray(options.order)) {
+            // We need to make sure the order by attributes are available to the parent query
+            options.order.forEach((order, i) => {
+              if (Array.isArray(order)) {
+                order = order[0];
+              }
+
+              let alias = `subquery_order_${i}`;
+              options.attributes.push([order, alias]);
+
+              // We don't want to prepend model name when we alias the attributes, so quote them here
+              alias = this.sequelize.literal(this.quote(alias));
+
+              if (Array.isArray(options.order[i])) {
+                options.order[i][0] = alias;
+              } else {
+                options.order[i] = alias;
+              }
+            });
+            groupedLimitOrder = options.order;
+          }
+        } else {
+          // Ordering is handled by the subqueries, so ordering the UNION'ed result is not needed
+          groupedLimitOrder = options.order;
+          delete options.order;
+          where.$$PLACEHOLDER$$ = true;
+        }
+
+        // Caching the base query and splicing the where part into it is consistently > twice
+        // as fast than generating from scratch each time for values.length >= 5
+        const baseQuery = '('+this.selectQuery(
+          tableName,
+          {
+            attributes: options.attributes,
+            limit: options.groupedLimit.limit,
+            order: groupedLimitOrder,
+            where,
+            include,
+            model
+          },
+          model
+        ).replace(/;$/, '')+')';
+        const placeHolder = this.whereItemQuery('$$PLACEHOLDER$$', true, { model });
+        const splicePos = baseQuery.indexOf(placeHolder);
+
         mainQueryItems.push(this.selectFromTableFragment(options, model, mainAttributes, '('+
         mainQueryItems.push(this.selectFromTableFragment(options, model, mainAttributes, '('+
           options.groupedLimit.values.map(value => {
           options.groupedLimit.values.map(value => {
-            const where = _.assign({}, options.where);
-            where[options.groupedLimit.on] = value;
-
-            return '('+this.selectQuery(
-              tableName,
-              {
-                attributes: options.attributes,
-                limit: options.groupedLimit.limit,
-                order: options.order,
-                where
-              },
-              model
-            ).replace(/;$/, '')+')';
+            let groupWhere;
+            if (whereKey) {
+              groupWhere = {
+                [whereKey]: value
+              };
+            }
+            if (include) {
+              groupWhere = {
+                [options.groupedLimit.on.otherKey]: value
+              };
+            }
+
+            return Utils.spliceStr(baseQuery, splicePos, placeHolder.length, this.getWhereConditions(groupWhere, groupedTableName));
           }).join(
           }).join(
             this._dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION '
             this._dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION '
           )
           )
@@ -1447,7 +1529,7 @@ const QueryGenerator = {
       }
       }
     }
     }
     // Add ORDER to sub or main query
     // Add ORDER to sub or main query
-    if (options.order && !options.groupedLimit) {
+    if (options.order) {
       const orders = this.getQueryOrders(options, model, subQuery);
       const orders = this.getQueryOrders(options, model, subQuery);
 
 
       if (orders.mainQueryOrder.length) {
       if (orders.mainQueryOrder.length) {
@@ -1551,7 +1633,7 @@ const QueryGenerator = {
     return {mainQueryOrder, subQueryOrder};
     return {mainQueryOrder, subQueryOrder};
   },
   },
 
 
-  selectFromTableFragment(options, model, attributes, tables, mainTableAs, whereClause) {
+  selectFromTableFragment(options, model, attributes, tables, mainTableAs) {
     let fragment = 'SELECT ' + attributes.join(', ') + ' FROM ' + tables;
     let fragment = 'SELECT ' + attributes.join(', ') + ' FROM ' + tables;
 
 
     if(mainTableAs) {
     if(mainTableAs) {

test/integration/model/findAll/groupedLimit.test.js

+'use strict';
+
+/* jshint -W030 */
+var chai = require('chai')
+  , expect = chai.expect
+  , Support = require(__dirname + '/../../support')
+  , Sequelize = Support.Sequelize
+  , DataTypes = require(__dirname + '/../../../../lib/data-types')
+  , current = Support.sequelize
+  , Promise = current.Promise
+  , _ = require('lodash');
+
+if (current.dialect.supports['UNION ALL']) {
+  describe(Support.getTestDialectTeaser('Model'), function() {
+    describe('findAll', function () {
+      describe('groupedLimit', function () {
+        beforeEach(function () {
+          this.User = this.sequelize.define('user', {
+            age: Sequelize.INTEGER
+          });
+          this.Project = this.sequelize.define('project', {
+            title: DataTypes.STRING
+          });
+          this.Task = this.sequelize.define('task');
+          this.User.Projects = this.User.belongsToMany(this.Project, {through: 'project_user' });
+          this.Project.belongsToMany(this.User, {as: 'members', through: 'project_user' });
+
+          this.User.Tasks = this.User.hasMany(this.Task);
+
+          return this.sequelize.sync({force: true}).then(() => {
+            return Promise.join(
+              this.User.bulkCreate([{age: -5}, {age: 45}, {age: 7}, {age: -9}, {age: 8}, {age: 15}, {age: -9}]),
+              this.Project.bulkCreate([{}, {}]),
+              this.Task.bulkCreate([{}, {}])
+            );
+          })
+            .then(() => [this.User.findAll(), this.Project.findAll(), this.Task.findAll()])
+            .spread((users, projects, tasks) => {
+              this.projects = projects;
+              return Promise.join(
+                projects[0].setMembers(users.slice(0, 4)),
+                projects[1].setMembers(users.slice(2)),
+                users[2].setTasks(tasks)
+              );
+            });
+        });
+
+        describe('on: belongsToMany', function () {
+          it('maps attributes from a grouped limit to models', function () {
+            return this.User.findAll({
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(item => item.get('id'))
+              }
+            }).then(users => {
+              expect(users).to.have.length(5);
+              users.filter(u => u.get('id') !== 3).forEach(u => {
+                expect(u.get('projects')).to.have.length(1);
+              });
+              users.filter(u => u.get('id') === 3).forEach(u => {
+                expect(u.get('projects')).to.have.length(2);
+              });
+            });
+          });
+
+          it('maps attributes from a grouped limit to models with include', function () {
+            return this.User.findAll({
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(item => item.get('id'))
+              },
+              order: ['id'],
+              include: [this.User.Tasks]
+            }).then(users => {
+              /*
+               project1 - 1, 2, 3
+               project2 - 3, 4, 5
+               */
+              expect(users).to.have.length(5);
+              expect(users.map(u => u.get('id'))).to.deep.equal([1, 2, 3, 4, 5]);
+
+              expect(users[2].get('tasks')).to.have.length(2);
+              users.filter(u => u.get('id') !== 3).forEach(u => {
+                expect(u.get('projects')).to.have.length(1);
+              });
+              users.filter(u => u.get('id') === 3).forEach(u => {
+                expect(u.get('projects')).to.have.length(2);
+              });
+            });
+          });
+
+          it('works with computed order', function () {
+            return this.User.findAll({
+              attributes: ['id'],
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(item => item.get('id'))
+              },
+              order: [
+                Sequelize.fn('ABS', Sequelize.col('age'))
+              ],
+              include: [this.User.Tasks]
+            }).then(users => {
+              /*
+               project1 - 1, 3, 4
+               project2 - 3, 5, 4
+             */
+              expect(users).to.have.length(4);
+              expect(users.map(u => u.get('id'))).to.deep.equal([1, 3, 5, 4]);
+            });
+          });
+
+          it('works with multiple orders', function () {
+            return this.User.findAll({
+              attributes: ['id'],
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(item => item.get('id'))
+              },
+              order: [
+                Sequelize.fn('ABS', Sequelize.col('age')),
+                ['id', 'DESC']
+              ],
+              include: [this.User.Tasks]
+            }).then(users => {
+              /*
+                project1 - 1, 3, 4
+                project2 - 3, 5, 7
+               */
+              expect(users).to.have.length(5);
+              expect(users.map(u => u.get('id'))).to.deep.equal([1, 3, 5, 7, 4]);
+            });
+          });
+        });
+
+        describe('on: hasMany', function () {
+          beforeEach(function () {
+            this.User = this.sequelize.define('user');
+            this.Task = this.sequelize.define('task');
+            this.User.Tasks = this.User.hasMany(this.Task);
+
+            return this.sequelize.sync({force: true}).then(() => {
+              return Promise.join(
+                this.User.bulkCreate([{}, {}, {}]),
+                this.Task.bulkCreate([{id: 1}, {id: 2}, {id: 3}, {id: 4}, {id: 5}, {id: 6}])
+              );
+            })
+              .then(() => [this.User.findAll(), this.Task.findAll()])
+              .spread((users, tasks) => {
+                this.users = users;
+                return Promise.join(
+                  users[0].setTasks(tasks[0]),
+                  users[1].setTasks(tasks.slice(1, 4)),
+                  users[2].setTasks(tasks.slice(4))
+                );
+              });
+          });
+
+          it('Applies limit and order correctly', function () {
+            return this.Task.findAll({
+              order: [
+                ['id', 'DESC']
+              ],
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Tasks,
+                values: this.users.map(item => item.get('id'))
+              }
+            }).then(tasks => {
+              const byUser = _.groupBy(tasks, _.property('userId'));
+              expect(Object.keys(byUser)).to.have.length(3);
+
+              expect(byUser[1]).to.have.length(1);
+              expect(byUser[2]).to.have.length(3);
+              expect(_.invokeMap(byUser[2], 'get', 'id')).to.deep.equal([4, 3, 2]);
+              expect(byUser[3]).to.have.length(2);
+            });
+          });
+        });
+      });
+    });
+  });
+}

test/unit/sql/select.test.js

@@ -73,6 +73,82 @@ suite(Support.getTestDialectTeaser('SQL'), function() {
       +') AS [User];'
       +') AS [User];'
     });
     });
 
 
+    (function() {
+      const User = Support.sequelize.define('user', {
+        id: {
+          type: DataTypes.INTEGER,
+          primaryKey: true,
+          autoIncrement: true,
+          field: 'id_user'
+        }
+      });
+      const Project = Support.sequelize.define('project', {
+        title: DataTypes.STRING
+      });
+
+      const ProjectUser = Support.sequelize.define('project_user', {}, { timestamps: false });
+
+      User.Projects = User.belongsToMany(Project, { through: ProjectUser });
+      Project.belongsToMany(User, { through: ProjectUser });
+
+      testsql({
+        table: User.getTableName(),
+        model: User,
+        attributes: [
+          ['id_user', 'id']
+        ],
+        order: [
+          ['last_name', 'ASC']
+        ],
+        groupedLimit: {
+          limit: 3,
+          on: User.Projects,
+          values: [
+            1,
+            5
+          ]
+        }
+      }, {
+        default: 'SELECT [user].* FROM ('+
+          [
+            '(SELECT [user].[id_user] AS [id], [user].[last_name] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 1 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [user].[id_user]' : '') + sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')',
+            '(SELECT [user].[id_user] AS [id], [user].[last_name] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 5 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [user].[id_user]' : '') +sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')'
+          ].join(current.dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION ')
+        +') AS [user] ORDER BY [subquery_order_0] ASC;'
+      });
+
+      testsql({
+        table: User.getTableName(),
+        model: User,
+        attributes: [
+          ['id_user', 'id']
+        ],
+        order: [
+          ['id_user', 'ASC']
+        ],
+        where: {
+          age: {
+            $gte: 21
+          }
+        },
+        groupedLimit: {
+          limit: 3,
+          on: User.Projects,
+          values: [
+            1,
+            5
+          ]
+        }
+      }, {
+        default: 'SELECT [user].* FROM ('+
+          [
+            '(SELECT [user].[id_user] AS [id], [user].[id_user] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 1 WHERE [user].[age] >= 21 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [user].[id_user]' : '') + sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')',
+            '(SELECT [user].[id_user] AS [id], [user].[id_user] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 5 WHERE [user].[age] >= 21 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [user].[id_user]' : '') +sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')'
+          ].join(current.dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION ')
+        +') AS [user] ORDER BY [subquery_order_0] ASC;'
+      });
+    }());
+
     (function () {
     (function () {
       var User = Support.sequelize.define('user', {
       var User = Support.sequelize.define('user', {
         id: {
         id: {