feat(validations) refactor(query-gen) Add context: UPDATE to bulk update. Refact…

…or bulk update for mysql, sqlite and pg into abstract qg

changelog.md

 # Next
-- [FIXED] Partial rollback of datatype validations by hiding it behind the `validation` flag.
-
+- [FIXED] Partial rollback of datatype validations by hiding it behind the `typeValidation` flag.
 
 # 3.11.0
 - [INTERNALS] Updated dependencies [#4594](https://github.com/sequelize/sequelize/pull/4594)

lib/dialects/abstract/index.js

@@ -30,7 +30,12 @@ AbstractDialect.prototype.supports = {
     /* does the dialect support updating autoincrement fields */
     update: true
   },
-
+  /* Do we need to say DEFAULT for bulk insert */
+  bulkDefault: false,
+  /* The dialect's words for INSERT IGNORE */
+  ignoreDuplicates: '',
+  /* Does the dialect support ON DUPLICATE KEY UPDATE */
+  updateOnDuplicate: false,
   schemas: false,
   transactions: true,
   migrations: true,

lib/dialects/abstract/query-generator.js

@@ -314,13 +314,64 @@ var QueryGenerator = {
 
     return Utils._.template(query)(replacements);
   },
+
   /*
     Returns an insert into command for multiple values.
     Parameters: table name + list of hashes of attribute-value-pairs.
   */
-  /* istanbul ignore next */
-  bulkInsertQuery: function(tableName, attrValueHashes, options, modelAttributes) {
-    throwMethodUndefined('bulkInsertQuery');
+  bulkInsertQuery: function(tableName, attrValueHashes, options, rawAttributes) {
+    options = options || {};
+    rawAttributes = rawAttributes || {};
+
+    var query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %><%= returning %>;'
+      , tuples = []
+      , serials = []
+      , allAttributes = []
+      , onDuplicateKeyUpdate = '';
+
+    attrValueHashes.forEach(function(attrValueHash) {
+      _.forOwn(attrValueHash, function(value, key) {
+        if (allAttributes.indexOf(key) === -1) {
+          allAttributes.push(key);
+        }
+
+        if (rawAttributes[key] && rawAttributes[key].autoIncrement === true) {
+          serials.push(key);
+        }
+      });
+    });
+
+    attrValueHashes.forEach(function(attrValueHash) {
+      tuples.push('(' +
+        allAttributes.map(function(key) {
+          if (this._dialect.supports.bulkDefault && serials.indexOf(key) !== -1) {
+            return attrValueHash[key] || 'DEFAULT';
+          }
+          return this.escape(attrValueHash[key], rawAttributes[key], { context: 'INSERT' });
+        }, this).join(',') +
+        ')');
+    }, this);
+
+    if (this._dialect.supports.updateOnDuplicate && options.updateOnDuplicate) {
+      onDuplicateKeyUpdate += ' ON DUPLICATE KEY UPDATE ' + options.updateOnDuplicate.map(function(attr) {
+        var field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr;
+        var key = this.quoteIdentifier(field);
+        return key + '=VALUES(' + key + ')';
+      }, this).join(',');
+    }
+
+    var replacements = {
+      ignoreDuplicates: options.ignoreDuplicates ? this._dialect.supports.ignoreDuplicates : '',
+      table: this.quoteTable(tableName),
+      attributes: allAttributes.map(function(attr) {
+        return this.quoteIdentifier(attr);
+      }, this).join(','),
+      tuples: tuples.join(','),
+      onDuplicateKeyUpdate: onDuplicateKeyUpdate,
+      returning: this._dialect.supports.returnValues && options.returning ? ' RETURNING *' : ''
+    };
+
+    return _.template(query)(replacements);
   },
 
   /*
@@ -899,7 +950,7 @@ var QueryGenerator = {
     if (value && value._isSequelizeMethod) {
       return this.handleSequelizeMethod(value);
     } else {
-      if (['INSERT', 'UPDATE'].indexOf(options.context) !== -1 &&this.validation && field && field.type && value) {
+      if (['INSERT', 'UPDATE'].indexOf(options.context) !== -1 && this.typeValidation && field && field.type && value) {
         if (field.type.validate) {
           field.type.validate(value);
         }

lib/dialects/mysql/index.js

@@ -31,6 +31,8 @@ MysqlDialect.prototype.supports = _.merge(_.cloneDeep(Abstract.prototype.support
     type: true,
     using: 1,
   },
+  ignoreDuplicates: ' IGNORE',
+  updateOnDuplicate: true,
   indexViaAlter: true,
   NUMERIC: true,
   GEOMETRY: true

lib/dialects/mysql/query-generator.js

@@ -167,47 +167,6 @@ var QueryGenerator = {
     return this.insertQuery(tableName, insertValues, rawAttributes, options);
   },
 
-  bulkInsertQuery: function(tableName, attrValueHashes, options, rawAttributes) {
-    var query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %><%= onDuplicateKeyUpdate %>;'
-      , tuples = []
-      , allAttributes = []
-      , onDuplicateKeyUpdate = '';
-
-    Utils._.forEach(attrValueHashes, function(attrValueHash) {
-      Utils._.forOwn(attrValueHash, function(value, key) {
-        if (allAttributes.indexOf(key) === -1) allAttributes.push(key);
-      });
-    });
-
-    Utils._.forEach(attrValueHashes, function(attrValueHash) {
-      tuples.push('(' +
-        allAttributes.map(function(key) {
-          return this.escape(attrValueHash[key]);
-        }.bind(this)).join(',') +
-      ')');
-    }.bind(this));
-
-    if (options && options.updateOnDuplicate) {
-      onDuplicateKeyUpdate += ' ON DUPLICATE KEY UPDATE ' + options.updateOnDuplicate.map(function(attr) {
-        var field = rawAttributes && rawAttributes[attr] && rawAttributes[attr].field || attr;
-        var key = this.quoteIdentifier(field);
-        return key + '=VALUES(' + key + ')';
-      }.bind(this)).join(',');
-    }
-
-    var replacements = {
-      ignoreDuplicates: options && options.ignoreDuplicates ? ' IGNORE' : '',
-      table: this.quoteTable(tableName),
-      attributes: allAttributes.map(function(attr) {
-                    return this.quoteIdentifier(attr);
-                  }.bind(this)).join(','),
-      tuples: tuples,
-      onDuplicateKeyUpdate: onDuplicateKeyUpdate
-    };
-
-    return Utils._.template(query)(replacements);
-  },
-
   deleteQuery: function(tableName, where, options) {
     options = options || {};
 
@@ -361,7 +320,7 @@ var QueryGenerator = {
     } else if (value && field && field.type instanceof DataTypes.GEOMETRY) {
       return 'GeomFromText(\'' + Wkt.stringify(value) + '\')';
     } else {
-      if (['INSERT', 'UPDATE'].indexOf(options.context) !== -1  && this.validation && field && field.type && value) {
+      if (['INSERT', 'UPDATE'].indexOf(options.context) !== -1  && this.typeValidation && field && field.type && value) {
         if (field.type.validate) {
           field.type.validate(value);
         }

lib/dialects/postgres/index.js

@@ -30,6 +30,7 @@ PostgresDialect.prototype.supports = _.merge(_.cloneDeep(Abstract.prototype.supp
   returnValues: {
     returning: true
   },
+  bulkDefault: true,
   schemas: true,
   lock: true,
   lockOf: true,

lib/dialects/postgres/query-generator.js

@@ -328,54 +328,6 @@ var QueryGenerator = {
     );
   },
 
-  bulkInsertQuery: function(tableName, attrValueHashes, options, modelAttributes) {
-    options = options || {};
-
-    var query = 'INSERT INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %>'
-      , tuples = []
-      , serials = []
-      , allAttributes = [];
-
-    if (options.returning) {
-      query += ' RETURNING *';
-    }
-
-    Utils._.forEach(attrValueHashes, function(attrValueHash) {
-      Utils._.forOwn(attrValueHash, function(value, key) {
-        if (allAttributes.indexOf(key) === -1) {
-          allAttributes.push(key);
-        }
-
-        if (modelAttributes && modelAttributes[key] && modelAttributes[key].autoIncrement === true) {
-          serials.push(key);
-        }
-      });
-    });
-
-    Utils._.forEach(attrValueHashes, function(attrValueHash) {
-      tuples.push('(' +
-        allAttributes.map(function(key) {
-          if (serials.indexOf(key) !== -1) {
-            return attrValueHash[key] || 'DEFAULT';
-          }
-          return this.escape(attrValueHash[key], modelAttributes && modelAttributes[key]);
-        }.bind(this)).join(',') +
-      ')');
-    }.bind(this));
-
-    var replacements = {
-      table: this.quoteTable(tableName)
-    , attributes: allAttributes.map(function(attr) {
-                    return this.quoteIdentifier(attr);
-                  }.bind(this)).join(',')
-    , tuples: tuples.join(',')
-    };
-
-    query = query + ';';
-
-    return Utils._.template(query)(replacements);
-  },
-
   deleteQuery: function(tableName, where, options, model) {
     var query;
 
@@ -902,7 +854,7 @@ var QueryGenerator = {
       return this.handleSequelizeMethod(value);
     }
 
-    if (['INSERT', 'UPDATE'].indexOf(options.context) !== -1 && this.validation && field && field.type && value) {
+    if (['INSERT', 'UPDATE'].indexOf(options.context) !== -1 && this.typeValidation && field && field.type && value) {
       if (field.type.validate) {
         field.type.validate(value);
       }

lib/dialects/sqlite/index.js

@@ -26,7 +26,8 @@ SqliteDialect.prototype.supports = _.merge(_.cloneDeep(Abstract.prototype.suppor
     using: false
   },
   joinTableDependent: false,
-  groupedLimit: false
+  groupedLimit: false,
+  ignoreDuplicates: ' OR IGNORE'
 });
 
 ConnectionManager.prototype.defaultVersion = '3.8.0';

lib/dialects/sqlite/query-generator.js

@@ -137,37 +137,6 @@ var QueryGenerator = {
     return sql;
   },
 
-  bulkInsertQuery: function(tableName, attrValueHashes, options) {
-    var query = 'INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %>;'
-      , tuples = []
-      , allAttributes = [];
-
-    Utils._.forEach(attrValueHashes, function(attrValueHash) {
-      Utils._.forOwn(attrValueHash, function(value, key) {
-        if (allAttributes.indexOf(key) === -1) allAttributes.push(key);
-      });
-    });
-
-    Utils._.forEach(attrValueHashes, function(attrValueHash) {
-      tuples.push('(' +
-        allAttributes.map(function (key) {
-          return this.escape(attrValueHash[key]);
-        }.bind(this)).join(',') +
-      ')');
-    }.bind(this));
-
-    var replacements  = {
-      ignoreDuplicates: options && options.ignoreDuplicates ? ' OR IGNORE' : '',
-      table: this.quoteTable(tableName),
-      attributes: allAttributes.map(function(attr) {
-                    return this.quoteIdentifier(attr);
-                  }.bind(this)).join(','),
-      tuples: tuples
-    };
-
-    return Utils._.template(query)(replacements);
-  },
-
   updateQuery: function(tableName, attrValueHash, where, options, attributes) {
     options = options || {};
     _.defaults(options, this.options);

lib/sequelize.js

@@ -80,7 +80,7 @@ var url = require('url')
  * @param {Function} [options.pool.validateConnection] A function that validates a connection. Called with client. The default function checks that client is an object, and that its state is not disconnected
  * @param {Boolean}  [options.quoteIdentifiers=true] Set to `false` to make table names and attributes case-insensitive on Postgres and skip double quoting of them.
  * @param {String}   [options.isolationLevel='REPEATABLE_READ'] Set the default transaction isolation level. See `Sequelize.Transaction.ISOLATION_LEVELS` for possible options.
- * @param {Boolean   [options.validation=false] Run built in type validators on insert and update, e.g. validate that arguments passed to integer fields are integer-like
+ * @param {Boolean   [options.typeValidation=false] Run built in type validators on insert and update, e.g. validate that arguments passed to integer fields are integer-like
 */
 
 /**
@@ -145,7 +145,7 @@ var Sequelize = function(database, username, password, options) {
     hooks: {},
     isolationLevel: Transaction.ISOLATION_LEVELS.REPEATABLE_READ,
     databaseVersion: 0,
-    validation: false
+    typeValidation: false
   }, options || {});
 
   if (this.options.dialect === 'postgresql') {
@@ -205,7 +205,7 @@ var Sequelize = function(database, username, password, options) {
     throw new Error('The dialect ' + this.getDialect() + ' is not supported. ('+err+')');
   }
 
-  this.dialect.QueryGenerator.validation = options.validation;
+  this.dialect.QueryGenerator.typeValidation = options.typeValidation;
 
   /**
    * Models are stored here under the name given to `sequelize.define`

test/integration/sequelize.test.js

@@ -945,6 +945,10 @@ describe(Support.getTestDialectTeaser('Sequelize'), function() {
     ].forEach(function(status) {
       describe('enum', function() {
         beforeEach(function() {
+          this.sequelize = Support.createSequelizeInstance({
+            typeValidation: true
+          });
+
           this.Review = this.sequelize.define('review', { status: status });
           return this.Review.sync({ force: true });
         });

test/unit/model/validation.test.js

@@ -270,7 +270,7 @@ describe(Support.getTestDialectTeaser('InstanceValidator'), function() {
 
   describe('datatype validations', function () {
     current = Support.createSequelizeInstance({
-      validation: true
+      typeValidation: true
     });
 
     var User = current.define('user', {
@@ -307,6 +307,26 @@ describe(Support.getTestDialectTeaser('InstanceValidator'), function() {
           })).not.to.be.rejected;
         });
 
+        it('should allow decimal big numbers as a string', function () {
+          return expect(User.create({
+            number: '2321312301230128391820831289123012'
+          })).not.to.be.rejected;
+        });
+
+        it('should allow decimal as scientific notation', function () {
+          return Promise.join(
+            expect(User.create({
+              number: '2321312301230128391820e219'
+            })).not.to.be.rejected,
+            expect(User.create({
+              number: '2321312301230128391820e+219'
+            })).not.to.be.rejected,
+            expect(User.create({
+              number: '2321312301230128391820f219'
+            })).to.be.rejected
+          );
+        });
+
         it('should allow string as a number', function () {
           return expect(User.create({
             name: 12