Grouped limit include for v3 (#6588)

* Backport of #6560

* Test for multiple orders

* Fix test

* Use the correct alias

.travis.yml

@@ -39,3 +39,9 @@ matrix:
   allow_failures:
   allow_failures:
     - node_js: "0.10"
     - node_js: "0.10"
       env: COVERAGE=true
       env: COVERAGE=true
+
+branches:
+  only:
+  - master
+  - v3
+

lib/dialects/abstract/query-generator.js

@@ -8,6 +8,8 @@ var Utils = require('../../utils')
   , util = require('util')
   , util = require('util')
   , Dottie = require('dottie')
   , Dottie = require('dottie')
   , BelongsTo = require('../../associations/belongs-to')
   , BelongsTo = require('../../associations/belongs-to')
+  , BelongsToMany = require('../../associations/belongs-to-many')
+  , HasMany = require('../../associations/has-many')
   , uuid = require('node-uuid')
   , uuid = require('node-uuid')
   , semver = require('semver');
   , semver = require('semver');
 
 
@@ -1421,22 +1423,97 @@ var QueryGenerator = {
         if (!mainTableAs) {
         if (!mainTableAs) {
           mainTableAs = table;
           mainTableAs = table;
         }
         }
-        mainQueryItems.push(this.selectFromTableFragment(options, model, mainAttributes, '('+
-          options.groupedLimit.values.map(function (value) {
-            var where = _.assign({}, options.where);
-            where[options.groupedLimit.on] = value;
 
 
-            return '('+self.selectQuery(
-              table,
+        var groupedLimitOrder
+          , where = _.assign({}, options.where)
+          , whereKey
+          , include
+          , groupedTableName = mainTableAs;
+
+        if (typeof options.groupedLimit.on === 'string') {
+          whereKey = options.groupedLimit.on;
+        } else if (options.groupedLimit.on instanceof HasMany) {
+          whereKey = options.groupedLimit.on.foreignKeyField;
+        }
+
+        if (options.groupedLimit.on instanceof BelongsToMany) {
+          // BTM includes needs to join the through table on to check ID
+          groupedTableName = options.groupedLimit.on.manyFromSource.as;
+          var groupedLimitOptions = Model.$validateIncludedElements({
+            include: [{
+              association: options.groupedLimit.on.manyFromSource,
+              duplicating: false, // The UNION'ed query may contain duplicates, but each sub-query cannot
+              required: true,
+              where: {
+                '$$PLACEHOLDER$$': true
+              }
+            }],
+            model: model
+          });
+          // Make sure attributes from the join table are mapped back to models
+          options.hasJoin = true;
+          options.hasMultiAssociation = true;
+          options.includeMap = _.assign(groupedLimitOptions.includeMap, options.includeMap);
+          options.includeNames = groupedLimitOptions.includeNames.concat(options.includeNames || []);
+          include = groupedLimitOptions.include;
+
+          if (Array.isArray(options.order)) {
+            // We need to make sure the order by attributes are available to the parent query
+            options.order.forEach(function(order, i) {
+              if (Array.isArray(order)) {
+                order = order[0];
+              }
+
+              var alias = 'subquery_order_' +i;
+              options.attributes.push([order, alias]);
+
+              // We don't want to prepend model name when we alias the attributes, so quote them here
+              alias = this.sequelize.literal(this.quote(alias));
+
+              if (Array.isArray(options.order[i])) {
+                options.order[i][0] = alias;
+              } else {
+                options.order[i] = alias;
+              }
+            }, this);
+            groupedLimitOrder = options.order;
+          }
+        } else {
+          // Ordering is handled by the subqueries, so ordering the UNION'ed result is not needed
+          groupedLimitOrder = options.order;
+          delete options.order;
+          where.$$PLACEHOLDER$$ = true;
+        }
+
+        // Caching the base query and splicing the where part into it is consistently > twice
+        // as fast than generating from scratch each time for values.length >= 5
+        var baseQuery = '('+this.selectQuery(
+          tableName,
           {
           {
             attributes: options.attributes,
             attributes: options.attributes,
             limit: options.groupedLimit.limit,
             limit: options.groupedLimit.limit,
-                order: options.order,
-                where: where
+            order: groupedLimitOrder,
+            where: where,
+            include: include,
+            model: model
           },
           },
           model
           model
         ).replace(/;$/, '')+')';
         ).replace(/;$/, '')+')';
-          }).join(
+        var placeHolder = this.whereItemQuery('$$PLACEHOLDER$$', true, { model: model })
+          , splicePos = baseQuery.indexOf(placeHolder);
+
+        mainQueryItems.push(this.selectFromTableFragment(options, model, mainAttributes, '('+
+          options.groupedLimit.values.map(function (value) {
+            var groupWhere = {};
+            if (whereKey) {
+              groupWhere[whereKey] = value;
+            }
+            if (include) {
+              groupWhere[options.groupedLimit.on.otherKey] = value;
+            }
+
+            return Utils.spliceStr(baseQuery, splicePos, placeHolder.length, this.getWhereConditions(groupWhere, groupedTableName));
+          }, this).join(
             self._dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION '
             self._dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION '
           )
           )
         +')', mainTableAs));
         +')', mainTableAs));
@@ -1484,7 +1561,7 @@ var QueryGenerator = {
       }
       }
     }
     }
     // Add ORDER to sub or main query
     // Add ORDER to sub or main query
-    if (options.order && !options.groupedLimit) {
+    if (options.order) {
       var orders = this.getQueryOrders(options, model, subQuery);
       var orders = this.getQueryOrders(options, model, subQuery);
 
 
       if (orders.mainQueryOrder.length) {
       if (orders.mainQueryOrder.length) {

test/integration/model/findAll/groupedLimit.test.js

+'use strict';
+
+/* jshint -W030 */
+var chai = require('chai')
+  , expect = chai.expect
+  , Support = require(__dirname + '/../../support')
+  , Sequelize = Support.Sequelize
+  , DataTypes = require(__dirname + '/../../../../lib/data-types')
+  , current = Support.sequelize
+  , Promise = current.Promise
+  , _ = require('lodash');
+
+if (current.dialect.supports['UNION ALL']) {
+  describe(Support.getTestDialectTeaser('Model'), function() {
+    describe('findAll', function () {
+      describe('groupedLimit', function () {
+        beforeEach(function () {
+          this.User = this.sequelize.define('user', {
+            age: Sequelize.INTEGER
+          });
+          this.Project = this.sequelize.define('project', {
+            title: DataTypes.STRING
+          });
+          this.Task = this.sequelize.define('task');
+
+          this.ProjectUser = this.sequelize.define('project_user', {}, {timestamps: false});
+
+          this.User.Projects = this.User.belongsToMany(this.Project, {through: this.ProjectUser});
+          this.Project.belongsToMany(this.User, {as: 'members', through: this.ProjectUser});
+
+          this.User.Tasks = this.User.hasMany(this.Task);
+
+          return this.sequelize.sync({force: true}).bind(this).then(function() {
+            return Promise.join(
+              this.User.bulkCreate([{age: -5}, {age: 45}, {age: 7}, {age: -9}, {age: 8}, {age: 15}, {age: -9}]),
+              this.Project.bulkCreate([{}, {}]),
+              this.Task.bulkCreate([{}, {}])
+            );
+          })
+            .then(function() { return [this.User.findAll(), this.Project.findAll(), this.Task.findAll()]; })
+            .spread(function (users, projects, tasks) {
+              this.projects = projects;
+              return Promise.join(
+                projects[0].setMembers(users.slice(0, 4)),
+                projects[1].setMembers(users.slice(2)),
+                users[2].setTasks(tasks)
+              );
+            });
+        });
+
+        describe('on: belongsToMany', function () {
+          it('maps attributes from a grouped limit to models', function () {
+            return this.User.findAll({
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(function (item) { return item.get('id'); })
+              }
+            }).then(function (users) {
+              expect(users).to.have.length(5);
+              users.filter(function (u) { return u.get('id') !== 3; }).forEach(function (u) {
+                expect(u.get('project_users')).to.have.length(1);
+              });
+              users.filter(function (u) { return u.get('id') === 3; }).forEach(function(u) {
+                expect(u.get('project_users')).to.have.length(2);
+              });
+            });
+          });
+
+          it('maps attributes from a grouped limit to models with include', function () {
+            return this.User.findAll({
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(function (item) { return item.get('id'); })
+              },
+              order: ['id'],
+              include: [this.User.Tasks]
+            }).then(function (users) {
+              /*
+               project1 - 1, 2, 3
+               project2 - 3, 4, 5
+               */
+              expect(users).to.have.length(5);
+
+              expect(users.map(function (u) { return u.get('id'); })).to.deep.equal([1, 2, 3, 4, 5]);
+
+              expect(users[2].get('tasks')).to.have.length(2);
+              users.filter(function (u) { return u.get('id') !== 3; }).forEach(function (u) {
+                expect(u.get('project_users')).to.have.length(1);
+              });
+              users.filter(function (u) { return u.get('id') === 3; }).forEach(function(u) {
+                expect(u.get('project_users')).to.have.length(2);
+              });
+            });
+          });
+
+          it('works with computed order', function () {
+            return this.User.findAll({
+              attributes: ['id'],
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(function (item) { return item.get('id'); })
+              },
+              order: [
+                Sequelize.fn('ABS', Sequelize.col('age'))
+              ],
+              include: [this.User.Tasks]
+            }).then(function (users) {
+              /*
+               project1 - 1, 3, 4
+               project2 - 3, 5, 4
+             */
+              expect(users).to.have.length(4);
+              expect(users.map(function(u) { return u.get('id'); })).to.deep.equal([1, 3, 5, 4]);
+            });
+          });
+
+          it('works with multiple orders', function () {
+            return this.User.findAll({
+              attributes: ['id'],
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Projects,
+                values: this.projects.map(function (item) { return item.get('id'); })
+              },
+              order: [
+                Sequelize.fn('ABS', Sequelize.col('age')),
+                ['id', 'DESC']
+              ],
+              include: [this.User.Tasks]
+            }).then(function (users) {
+              /*
+                project1 - 1, 3, 4
+                project2 - 3, 5, 7
+               */
+              expect(users).to.have.length(5);
+              expect(users.map(function (u) { return u.get('id'); })).to.deep.equal([1, 3, 5, 7, 4]);
+            });
+          });
+        });
+
+        describe('on: hasMany', function () {
+          beforeEach(function () {
+            this.User = this.sequelize.define('user');
+            this.Task = this.sequelize.define('task');
+            this.User.Tasks = this.User.hasMany(this.Task);
+
+            return this.sequelize.sync({force: true}).bind(this).then(function() {
+              return Promise.join(
+                this.User.bulkCreate([{}, {}, {}]),
+                this.Task.bulkCreate([{id: 1}, {id: 2}, {id: 3}, {id: 4}, {id: 5}, {id: 6}])
+              );
+            })
+              .then(function() { return [this.User.findAll(), this.Task.findAll()]; })
+              .spread(function(users, tasks) {
+                this.users = users;
+                return Promise.join(
+                  users[0].setTasks(tasks[0]),
+                  users[1].setTasks(tasks.slice(1, 4)),
+                  users[2].setTasks(tasks.slice(4))
+                );
+              });
+          });
+
+          it('Applies limit and order correctly', function () {
+            return this.Task.findAll({
+              order: [
+                ['id', 'DESC']
+              ],
+              groupedLimit: {
+                limit: 3,
+                on: this.User.Tasks,
+                values: this.users.map(function (item) { return item.get('id'); })
+              }
+            }).then(function (tasks) {
+              var byUser = _.groupBy(tasks, _.property('userId'));
+              expect(Object.keys(byUser)).to.have.length(3);
+
+              expect(byUser[1]).to.have.length(1);
+              expect(byUser[2]).to.have.length(3);
+              expect(_.invokeMap(byUser[2], 'get', 'id')).to.deep.equal([4, 3, 2]);
+              expect(byUser[3]).to.have.length(2);
+            });
+          });
+        });
+      });
+    });
+  });
+}

test/unit/sql/select.test.js

@@ -73,6 +73,82 @@ suite(Support.getTestDialectTeaser('SQL'), function() {
       +') AS [User];'
       +') AS [User];'
     });
     });
 
 
+    (function() {
+      var User = Support.sequelize.define('user', {
+        id: {
+          type: DataTypes.INTEGER,
+          primaryKey: true,
+          autoIncrement: true,
+          field: 'id_user'
+        }
+      });
+      var Project = Support.sequelize.define('project', {
+        title: DataTypes.STRING
+      });
+
+      var ProjectUser = Support.sequelize.define('project_user', {}, { timestamps: false });
+
+      User.Projects = User.belongsToMany(Project, { through: ProjectUser });
+      Project.belongsToMany(User, { through: ProjectUser });
+
+      testsql({
+        table: User.getTableName(),
+        model: User,
+        attributes: [
+          ['id_user', 'id']
+        ],
+        order: [
+          ['last_name', 'ASC']
+        ],
+        groupedLimit: {
+          limit: 3,
+          on: User.Projects,
+          values: [
+            1,
+            5
+          ]
+        }
+      }, {
+        default: 'SELECT [user].* FROM ('+
+        [
+          '(SELECT [user].[id_user] AS [id], [user].[last_name] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 1 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [id_user]' : '') + sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')',
+          '(SELECT [user].[id_user] AS [id], [user].[last_name] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 5 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [id_user]' : '') +sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')'
+        ].join(current.dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION ')
+        +') AS [user] ORDER BY [subquery_order_0] ASC;'
+      });
+
+      testsql({
+        table: User.getTableName(),
+        model: User,
+        attributes: [
+          ['id_user', 'id']
+        ],
+        order: [
+          ['id_user', 'ASC']
+        ],
+        where: {
+          age: {
+            $gte: 21
+          }
+        },
+        groupedLimit: {
+          limit: 3,
+          on: User.Projects,
+          values: [
+            1,
+            5
+          ]
+        }
+      }, {
+        default: 'SELECT [user].* FROM ('+
+        [
+          '(SELECT [user].[id_user] AS [id], [user].[id_user] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 1 WHERE [user].[age] >= 21 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [id_user]' : '') + sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')',
+          '(SELECT [user].[id_user] AS [id], [user].[id_user] AS [subquery_order_0], [project_users].[userId] AS [project_users.userId], [project_users].[projectId] AS [project_users.projectId] FROM [users] AS [user] INNER JOIN [project_users] AS [project_users] ON [user].[id_user] = [project_users].[userId] AND [project_users].[projectId] = 5 WHERE [user].[age] >= 21 ORDER BY [subquery_order_0] ASC'+ (current.dialect.name === 'mssql' ? ', [id_user]' : '') +sql.addLimitAndOffset({ limit: 3, order: ['last_name', 'ASC'] })+')'
+        ].join(current.dialect.supports['UNION ALL'] ?' UNION ALL ' : ' UNION ')
+        +') AS [user] ORDER BY [subquery_order_0] ASC;'
+      });
+    }());
+
     (function () {
     (function () {
       var User = Support.sequelize.define('user', {
       var User = Support.sequelize.define('user', {
         id: {
         id: {