Support Large BulkInsert for MSSQL (#6904)

* fixes for MSSQL bulkInsetQuery unit test edge cases and add large integration test

* put back es6project.js

* Revert "temporary fork to add upsert to mssql until we upgrade sequelize"

This reverts commit 14d73a0a958efd981d51a9439579fd2b00698cfa.

* update changelog

* fix up changelog

* space - review comment

* use const in master

* es6 updates

* more es6 updates

* quotes

changelog.md

 # Future
 # Future
+- [FIXED] MSSQL handle large bulk inserts #6866
 - [FIXED] describeTable returns a wrong value for primaryKey [#5756] (https://github.com/sequelize/sequelize/issues/5756)
 - [FIXED] describeTable returns a wrong value for primaryKey [#5756] (https://github.com/sequelize/sequelize/issues/5756)
 - [FIXED] MSSQL LIMIT IN UPDATE [#6636](https://github.com/sequelize/sequelize/issues/6636)
 - [FIXED] MSSQL LIMIT IN UPDATE [#6636](https://github.com/sequelize/sequelize/issues/6636)
 - [FIXED] Custom error message not used for `notNull` validation [#6531](https://github.com/sequelize/sequelize/issues/6531)
 - [FIXED] Custom error message not used for `notNull` validation [#6531](https://github.com/sequelize/sequelize/issues/6531)

lib/dialects/mssql/query-generator.js

@@ -254,8 +254,8 @@ var QueryGenerator = {
 
 
     Utils._.forEach(attrValueHashes, function(attrValueHash) {
     Utils._.forEach(attrValueHashes, function(attrValueHash) {
       // special case for empty objects with primary keys
       // special case for empty objects with primary keys
-      var fields = Object.keys(attrValueHash);
-      var firstAttr = attributes[fields[0]];
+      let fields = Object.keys(attrValueHash);
+      let firstAttr = attributes[fields[0]];
       if (fields.length === 1 && firstAttr && firstAttr.autoIncrement && attrValueHash[fields[0]] === null) {
       if (fields.length === 1 && firstAttr && firstAttr.autoIncrement && attrValueHash[fields[0]] === null) {
         allQueries.push(emptyQuery);
         allQueries.push(emptyQuery);
         return;
         return;
@@ -279,33 +279,37 @@ var QueryGenerator = {
     if (allAttributes.length > 0) {
     if (allAttributes.length > 0) {
       Utils._.forEach(attrValueHashes, function(attrValueHash) {
       Utils._.forEach(attrValueHashes, function(attrValueHash) {
         tuples.push('(' +
         tuples.push('(' +
-          allAttributes.map(function(key) {
-            return this.escape(attrValueHash[key]);
-          }.bind(this)).join(',') +
+          allAttributes.map(key =>
+            this.escape(attrValueHash[key])).join(',') +
         ')');
         ')');
       }.bind(this));
       }.bind(this));
 
 
       allQueries.push(query);
       allQueries.push(query);
     }
     }
+    const commands = [];
+    let offset = 0;
+    const batch = Math.floor(250 / (allAttributes.length + 1)) + 1;
+    while (offset < Math.max(tuples.length, 1)) {
+      const replacements = {
+        table: this.quoteTable(tableName),
+        attributes: allAttributes.map(attr =>
+                      this.quoteIdentifier(attr)).join(','),
+        tuples: tuples.slice(offset, Math.min(tuples.length, offset + batch)),
+        output: outputFragment
+      };
 
 
-    var replacements = {
-      table: this.quoteTable(tableName),
-      attributes: allAttributes.map(function(attr) {
-        return this.quoteIdentifier(attr);
-      }.bind(this)).join(','),
-      tuples,
-      output: outputFragment
-    };
-
-    var generatedQuery = Utils._.template(allQueries.join(';'))(replacements);
-    if (needIdentityInsertWrapper) {
-      generatedQuery = [
-        'SET IDENTITY_INSERT', this.quoteTable(tableName), 'ON;',
-        generatedQuery,
-        'SET IDENTITY_INSERT', this.quoteTable(tableName), 'OFF;'
-      ].join(' ');
+      let generatedQuery = Utils._.template(allQueries.join(';'))(replacements);
+      if (needIdentityInsertWrapper) {
+        generatedQuery = [
+          'SET IDENTITY_INSERT', this.quoteTable(tableName), 'ON;',
+          generatedQuery,
+          'SET IDENTITY_INSERT', this.quoteTable(tableName), 'OFF;',
+        ].join(' ');
+      }
+      commands.push(generatedQuery);
+      offset += batch;
     }
     }
-    return generatedQuery;
+    return commands.join(';');
   },
   },
 
 
   updateQuery(tableName, attrValueHash, where, options, attributes) {
   updateQuery(tableName, attrValueHash, where, options, attributes) {

test/config/config.js

@@ -32,6 +32,11 @@ module.exports = {
     password: process.env.SEQ_MSSQL_PW   || process.env.SEQ_PW   || 'nEGkLma26gXVHFUAHJxcmsrK',
     password: process.env.SEQ_MSSQL_PW   || process.env.SEQ_PW   || 'nEGkLma26gXVHFUAHJxcmsrK',
     host:     process.env.SEQ_MSSQL_HOST || process.env.SEQ_HOST || 'mssql.sequelizejs.com',
     host:     process.env.SEQ_MSSQL_HOST || process.env.SEQ_HOST || 'mssql.sequelizejs.com',
     port:     process.env.SEQ_MSSQL_PORT || process.env.SEQ_PORT || 1433,
     port:     process.env.SEQ_MSSQL_PORT || process.env.SEQ_PORT || 1433,
+    dialectOptions: {
+      instanceName:  process.env.MSSQL_INSTANCE ? process.env.MSSQL_INSTANCE : 'SQLEXPRESS',
+      // big insert queries need a while
+      requestTimeout: 60000
+    },
     pool:     {
     pool:     {
       max: process.env.SEQ_MSSQL_POOL_MAX  || process.env.SEQ_POOL_MAX  || 5,
       max: process.env.SEQ_MSSQL_POOL_MAX  || process.env.SEQ_POOL_MAX  || 5,
       idle: process.env.SEQ_MSSQL_POOL_IDLE || process.env.SEQ_POOL_IDLE || 3000
       idle: process.env.SEQ_MSSQL_POOL_IDLE || process.env.SEQ_POOL_IDLE || 3000

test/integration/associations/has-many.test.js

@@ -10,6 +10,7 @@ var chai = require('chai')
   , sinon = require('sinon')
   , sinon = require('sinon')
   , Promise = Sequelize.Promise
   , Promise = Sequelize.Promise
   , current = Support.sequelize
   , current = Support.sequelize
+  , _ = require('lodash')
   , dialect = Support.getTestDialect();
   , dialect = Support.getTestDialect();
 
 
 describe(Support.getTestDialectTeaser('HasMany'), function() {
 describe(Support.getTestDialectTeaser('HasMany'), function() {
@@ -684,8 +685,26 @@ describe(Support.getTestDialectTeaser('HasMany'), function() {
           expect(users).to.have.length(3);
           expect(users).to.have.length(3);
         });
         });
       });
       });
-    });
 
 
+      it('handles decent sized bulk creates', function() {
+        const User = this.sequelize.define('User', { username: DataTypes.STRING, num: DataTypes.INTEGER, status: DataTypes.STRING })
+          , Task = this.sequelize.define('Task', { title: DataTypes.STRING });
+
+        Task.hasMany(User);
+
+        return this.sequelize.sync({ force: true }).then(() => {
+          const users = _.range(1000).map(i => ({username: 'user' + i, num: i, status: 'live'}));
+          return User.bulkCreate(users);
+        }).bind({}).then(() => {
+          return Task.create({ title: 'task' });
+        }).then(function(task) {
+          this.task = task;
+          return User.findAll();
+        }).then(users=> {
+          expect(users).to.have.length(1000);
+        });
+      });
+    });
     it('clears associations when passing null to the set-method with omitNull set to true', function() {
     it('clears associations when passing null to the set-method with omitNull set to true', function() {
       this.sequelize.options.omitNull = true;
       this.sequelize.options.omitNull = true;
 
 

test/support.js

@@ -97,7 +97,7 @@ var Support = {
       dialect: options.dialect,
       dialect: options.dialect,
       port: options.port || process.env.SEQ_PORT || config.port,
       port: options.port || process.env.SEQ_PORT || config.port,
       pool: config.pool,
       pool: config.pool,
-      dialectOptions: options.dialectOptions || {}
+      dialectOptions: options.dialectOptions || config.dialectOptions || {}
     });
     });
 
 
     if (process.env.DIALECT === 'postgres-native') {
     if (process.env.DIALECT === 'postgres-native') {