Merge branch 'master' into milestones/2.0.0

Mick Hansen
Commit a54e1c78 authored Feb 02, 2014 by Mick Hansen
Showing with 166 additions and 26 deletions
changelog.md
lib/associations/has-many.js
lib/dao-factory.js
lib/dialects/abstract/query-generator.js
lib/dialects/mysql/query-generator.js
lib/dialects/postgres/query-generator.js
lib/dialects/sqlite/query-generator.js
lib/query-interface.js
lib/utils.js
test/associations/has-many.test.js
test/dao-factory/create.test.js
test/dao-factory/findAll.test.js
test/mysql/query-generator.test.js
test/sqlite/query-generator.test.js
--- a/changelog.md
+++ b/changelog.md
@@ -5,6 +5,7 @@ Notice: All 1.7.x changes are present in 2.0.x aswell
 - fixes various issues with limit and includes [#1322](https://github.com/sequelize/sequelize/pull/1322) 
 - fixes issues with migrations/queryInterface createTable and enums
 - migration/queryInterface.addIndex() no longer fals on reserved keywords like 'from'
+- bulkCreate now supports a `ignoreDuplicates` option for MySQL, SQLite and MariaDB that will use `INSERT IGNORE`
 #### Backwards compatability changes
 - find/findAll will now always return primary keys regardless of `attributes` settings. (Motivation was to fix various issues with eager loading)

--- a/lib/associations/has-many.js
+++ b/lib/associations/has-many.js
@@ -19,7 +19,7 @@ module.exports = (function() {
    this.sequelize = source.daoFactoryManager.sequelize
    this.through = options.through
    this.isMultiAssociation = true
-    this.isSelfAssociation = (this.source.tableName === this.target.tableName)
+    this.isSelfAssociation = this.source === this.target
    this.doubleLinked = false
    this.combinedTableName = Utils.combineTableNames(
      this.source.tableName,
@@ -136,7 +136,7 @@ module.exports = (function() {
    // is there already a single sided association between the source and the target?
    // or is the association on the model itself?
-    if ((this.isSelfAssociation && this.through) || doubleLinked) {
+    if ((this.isSelfAssociation && Object(this.through) === this.through) || doubleLinked) {
      // remove the obsolete association identifier from the source
      if (this.isSelfAssociation) {
        this.foreignIdentifier = Utils._.underscoredIf((this.options.as || this.target.tableName) + 'Id', this.options.underscored)

--- a/lib/dao-factory.js
+++ b/lib/dao-factory.js
@@ -791,7 +791,8 @@ module.exports = (function() {
    options = Utils._.extend({
      validate: false,
-      hooks: false
+      hooks: false,
+      ignoreDuplicates: false
    }, options || {})
    if (fieldsOrOptions instanceof Array) {
@@ -801,6 +802,12 @@ module.exports = (function() {
      options = Utils._.extend(options, fieldsOrOptions)
    }
+    if(this.daoFactoryManager.sequelize.options.dialect === 'postgres' && options.ignoreDuplicates ) {
+      return new Utils.CustomEventEmitter(function(emitter) {
+        emitter.emit('error', new Error('Postgres does not support the \'ignoreDuplicates\' option.'))
+      }).run();
+    }
    var self          = this
      , updatedAtAttr = Utils._.underscoredIf(self.options.updatedAt, self.options.underscored)
      , createdAtAttr = Utils._.underscoredIf(self.options.createdAt, self.options.underscored)

--- a/lib/dialects/abstract/query-generator.js
+++ b/lib/dialects/abstract/query-generator.js
 var Utils = require("../../utils")
  , SqlString = require("../../sql-string")
+  , daoFactory = require("../../dao-factory")
 module.exports = (function() {
  var QueryGenerator = {
@@ -331,7 +332,14 @@ module.exports = (function() {
    /*
      Quote an object based on its type. This is a more general version of quoteIdentifiers
      Strings: should proxy to quoteIdentifiers
-      Arrays: First argument should be qouted, second argument should be append without quoting
+      Arrays:
+        * Expects array in the form: [<model> (optional), <model> (optional),... String, String (optional)]
+          Each <model> can be a daoFactory or an object {model: DaoFactory, as: String}, matching include
+        * Zero or more models can be included in the array and are used to trace a path through the tree of
+          included nested associations. This produces the correct table name for the ORDER BY/GROUP BY SQL
+          and quotes it.
+        * If a single string is appended to end of array, it is quoted.
+          If two strings appended, the 1st string is quoted, the 2nd string unquoted.
      Objects:
        * If raw is set, that value should be returned verbatim, without quoting
        * If fn is set, the string should start with the value of fn, starting paren, followed by
@@ -339,14 +347,71 @@ module.exports = (function() {
          unless they are themselves objects
        * If direction is set, should be prepended
-      Currently this function is only used for ordering / grouping columns, but it could
+      Currently this function is only used for ordering / grouping columns and Sequelize.col(), but it could
      potentially also be used for other places where we want to be able to call SQL functions (e.g. as default values)
    */
-    quote: function(obj, force) {
+    quote: function(obj, parent, force) {
      if (Utils._.isString(obj)) {
        return this.quoteIdentifiers(obj, force)
      } else if (Array.isArray(obj)) {
-        return this.quote(obj[0], force) + ' ' + obj[1]
+        // loop through array, adding table names of models to quoted
+        // (checking associations to see if names should be singularised or not)
+        var quoted = []
+          , i
+          , len = obj.length
+        for (i = 0; i < len - 1; i++) {
+          var item = obj[i]
+          if (Utils._.isString(item) || item instanceof Utils.fn || item instanceof Utils.col || item instanceof Utils.literal || item instanceof Utils.cast || 'raw' in item) {
+            break
+          }
+          if (item instanceof daoFactory) {
+            item = {model: item}
+          }
+          // find applicable association for linking parent to this model
+          var model = item.model
+            , as
+            , associations = parent.associations
+            , association
+          if (item.hasOwnProperty('as')) {
+            as = item.as
+            association = Utils._.find(associations, function(association, associationName) {
+              return association.target === model && associationName === as
+            })
+          } else {
+            association = Utils._.find(associations, function(association, associationName) {
+              return association.target === model ?
+                associationName === (
+                  association.doubleLinked ?
+                  association.combinedName:
+                  (
+                    association.isSingleAssociation ?
+                    Utils.singularize(model.tableName, model.options.language) :
+                    parent.tableName + model.tableName
+                  )
+                ) :
+                association.targetAssociation && association.targetAssociation.through === model
+            })
+            // NB association.target !== model clause below is to singularize names of through tables in hasMany-hasMany joins
+            as = (association && (association.isSingleAssociation || association.target !== model)) ? Utils.singularize(model.tableName, model.options.language) : model.tableName
+          }
+          quoted[i] = as
+          if (!association) {
+            throw new Error('\'' + quoted.join('.') + '\' in order / group clause is not valid association')
+          }
+          parent = model
+        }
+        // add 1st string as quoted, 2nd as unquoted raw
+        var sql = (i > 0 ? this.quoteIdentifier(quoted.join('.')) + '.' : '') + this.quote(obj[i], parent, force)
+        if (i < len - 1) {
+          sql += ' ' + obj[i + 1]
+        }
+        return sql
      } else if (obj instanceof Utils.fn || obj instanceof Utils.col || obj instanceof Utils.literal || obj instanceof Utils.cast) {
        return obj.toString(this)
      } else if (Utils._.isObject(obj) && 'raw' in obj) {
@@ -497,12 +562,12 @@ module.exports = (function() {
        }
        if (attr instanceof Utils.fn || attr instanceof Utils.col) {
-          return self.quote(attr)
+          return attr.toString(self)
        }
        if(Array.isArray(attr) && attr.length == 2) {
          if (attr[0] instanceof Utils.fn || attr[0] instanceof Utils.col) {
-            attr[0] = self.quote(attr[0])
+            attr[0] = attr[0].toString(self)
            addTable = false
          }
          attr = [attr[0], this.quoteIdentifier(attr[1])].join(' as ')
@@ -703,7 +768,7 @@ module.exports = (function() {
      // Add GROUP BY to sub or main query
      if (options.group) {
-        options.group = Array.isArray(options.group) ? options.group.map(function (t) { return this.quote(t) }.bind(this)).join(', ') : options.group
+        options.group = Array.isArray(options.group) ? options.group.map(function (t) { return this.quote(t, factory) }.bind(this)).join(', ') : options.group
        if (subQuery) {
          subQueryItems.push(" GROUP BY " + options.group)
        } else {
@@ -723,7 +788,7 @@ module.exports = (function() {
      // Add ORDER to sub or main query
      if (options.order) {
-        options.order = Array.isArray(options.order) ? options.order.map(function (t) { return this.quote(t) }.bind(this)).join(', ') : options.order
+        options.order = Array.isArray(options.order) ? options.order.map(function (t) { return this.quote(t, factory) }.bind(this)).join(', ') : options.order
        if (subQuery) {
          subQueryItems.push(" ORDER BY " + options.order)
@@ -950,9 +1015,9 @@ module.exports = (function() {
      })
      if (options.include) {
-        return this.quoteIdentifier(keyParts.join('.')) + '.' +  this.quote(attributePart);
+        return this.quoteIdentifier(keyParts.join('.')) + '.' + this.quoteIdentifiers(attributePart)
      }
-      return this.quoteIdentifiers(dao.tableName + '.' +  attributePart);
+      return this.quoteIdentifiers(dao.tableName + '.' + attributePart)
    },
    getConditionalJoins: function(options, originalDao){

--- a/lib/dialects/mysql/query-generator.js
+++ b/lib/dialects/mysql/query-generator.js
@@ -176,8 +176,8 @@ module.exports = (function() {
      return Utils._.template(query)({ tableName: tableName, attributes: attrString.join(', ') })
    },
-    bulkInsertQuery: function(tableName, attrValueHashes) {
+    bulkInsertQuery: function(tableName, attrValueHashes, options) {
-      var query = "INSERT INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %>;"
+      var query = "INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %>;"
        , tuples = []
        , allAttributes = []
@@ -196,6 +196,7 @@ module.exports = (function() {
      }.bind(this))
      var replacements  = {
+        ignoreDuplicates: options && options.ignoreDuplicates ? ' IGNORE' : '',
        table: this.quoteIdentifier(tableName),
        attributes: allAttributes.map(function(attr){
                      return this.quoteIdentifier(attr)

--- a/lib/dialects/postgres/query-generator.js
+++ b/lib/dialects/postgres/query-generator.js
@@ -267,7 +267,7 @@ module.exports = (function() {
      })
    },
-    bulkInsertQuery: function(tableName, attrValueHashes) {
+    bulkInsertQuery: function(tableName, attrValueHashes, options) {
      var query        = "INSERT INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %> RETURNING *;"
        , tuples       = []
        , serials      = []

--- a/lib/dialects/sqlite/query-generator.js
+++ b/lib/dialects/sqlite/query-generator.js
@@ -172,8 +172,8 @@ module.exports = (function() {
      return "SELECT name FROM sqlite_master WHERE type='table' and name!='sqlite_sequence';"
    },
-    bulkInsertQuery: function(tableName, attrValueHashes) {
+    bulkInsertQuery: function(tableName, attrValueHashes, options) {
-      var query = "INSERT INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %>;"
+      var query = "INSERT<%= ignoreDuplicates %> INTO <%= table %> (<%= attributes %>) VALUES <%= tuples %>;"
        , tuples = []
        , allAttributes = []
@@ -192,6 +192,7 @@ module.exports = (function() {
      }.bind(this))
      var replacements  = {
+        ignoreDuplicates: options && options.ignoreDuplicates ? ' OR IGNORE' : '',
        table: this.quoteIdentifier(tableName),
        attributes: allAttributes.map(function(attr){
                      return this.quoteIdentifier(attr)

--- a/lib/query-interface.js
+++ b/lib/query-interface.js
@@ -510,7 +510,7 @@ module.exports = (function() {
  }
  QueryInterface.prototype.bulkInsert = function(tableName, records, options) {
-    var sql = this.QueryGenerator.bulkInsertQuery(tableName, records)
+    var sql = this.QueryGenerator.bulkInsertQuery(tableName, records, options)
    return queryAndEmit.call(this, [sql, null, options], 'bulkInsert')
  }

--- a/lib/utils.js
+++ b/lib/utils.js
@@ -534,6 +534,9 @@ var Utils = module.exports = {
  },
  col: function (col) {
+    if (arguments.length > 1) {
+      col = Array.prototype.slice.call(arguments);
+    }
    this.col = col
  },
@@ -588,21 +591,25 @@ Utils.cast.prototype.toString = function(queryGenerator) {
  return 'CAST(' + this.val + ' AS ' + this.type.toUpperCase() + ')'
 }
-Utils.fn.prototype.toString = function(queryGenerator) {
+Utils.fn.prototype.toString = function(queryGenerator, parentModel) {
  return this.fn + '(' + this.args.map(function (arg) {
    if (arg instanceof Utils.fn || arg instanceof Utils.col) {
-      return arg.toString(queryGenerator)
+      return arg.toString(queryGenerator, parentModel)
    } else {
      return queryGenerator.escape(arg)
    }
  }).join(', ') + ')'
 }
-Utils.col.prototype.toString = function (queryGenerator) {
+Utils.col.prototype.toString = function (queryGenerator, parentModel) {
-  if (this.col.indexOf('*') === 0) {
+  if (Array.isArray(this.col)) {
+    if (!parent) {
+      throw new Error('Cannot call Sequelize.col() with array outside of order / group clause')
+    }
+  } else if (this.col.indexOf('*') === 0) {
    return '*'
  }
-  return queryGenerator.quote(this.col)
+  return queryGenerator.quote(this.col, parentModel)
 }
 Utils.CustomEventEmitter = require(__dirname + "/emitters/custom-event-emitter")

--- a/test/associations/has-many.test.js
+++ b/test/associations/has-many.test.js
@@ -518,7 +518,17 @@ describe(Support.getTestDialectTeaser("HasMany"), function() {
    }) // end optimization using bulk create, destroy and update
    describe('selfAssociations', function () {
-      it('should work', function (done) {
+      it('should work with alias', function (done) {
+        var Person = this.sequelize.define('Group', {})
+        Person.hasMany(Person, { as: 'Children'});
+        this.sequelize.sync().done(function (err) {
+          expect(err).not.to.be.ok
+          done()
+        })
+      })
+      it('should work with through', function (done) {
        var Group = this.sequelize.define('Group', {})
        Group.hasMany(Group, { through: 'groups_outsourcing_companies', as: 'OutsourcingCompanies'});

--- a/test/dao-factory/create.test.js
+++ b/test/dao-factory/create.test.js
@@ -24,7 +24,8 @@ describe(Support.getTestDialectTeaser("DAOFactory"), function () {
      data:         DataTypes.STRING,
      intVal:       DataTypes.INTEGER,
      theDate:      DataTypes.DATE,
-      aBool:        DataTypes.BOOLEAN
+      aBool:        DataTypes.BOOLEAN,
+      uniqueName:   { type: DataTypes.STRING, unique: true }
    })
    this.User.sync({ force: true }).success(function() {
@@ -1005,6 +1006,47 @@ describe(Support.getTestDialectTeaser("DAOFactory"), function () {
      })
    })
+    if (Support.getTestDialect() !== 'postgres') {
+      it("should support the ignoreDuplicates option", function(done) {
+        var self = this
+          , data = [{ uniqueName: 'Peter', secretValue: '42' },
+                    { uniqueName: 'Paul', secretValue: '23' }]
+        this.User.bulkCreate(data, { fields: ['uniqueName', 'secretValue'] }).success(function() {
+          data.push({ uniqueName: 'Michael', secretValue: '26' });
+          self.User.bulkCreate(data, { fields: ['uniqueName', 'secretValue'], ignoreDuplicates: true }).success(function() {
+            self.User.findAll({order: 'id'}).success(function(users) {
+              expect(users.length).to.equal(3)
+              expect(users[0].uniqueName).to.equal("Peter")
+              expect(users[0].secretValue).to.equal("42");
+              expect(users[1].uniqueName).to.equal("Paul")
+              expect(users[1].secretValue).to.equal("23");
+              expect(users[2].uniqueName).to.equal("Michael")
+              expect(users[2].secretValue).to.equal("26");
+              done()
+            });
+          });
+        })
+      })
+    } else {
+      it("should throw an error when the ignoreDuplicates option is passed", function(done) {
+        var self = this
+          , data = [{ uniqueName: 'Peter', secretValue: '42' },
+                    { uniqueName: 'Paul', secretValue: '23' }]
+        this.User.bulkCreate(data, { fields: ['uniqueName', 'secretValue'] }).success(function() {
+          data.push({ uniqueName: 'Michael', secretValue: '26' });
+          self.User.bulkCreate(data, { fields: ['uniqueName', 'secretValue'], ignoreDuplicates: true }).error(function(err) {
+            expect(err).to.exist
+            expect(err.message).to.match(/Postgres does not support the \'ignoreDuplicates\' option./)
+            done();
+          })
+        })
+      })
+    }
    describe('enums', function() {
      it('correctly restores enum values', function(done) {
        var self = this

--- a/test/dao-factory/findAll.test.js
+++ b/test/dao-factory/findAll.test.js
--- a/test/mysql/query-generator.test.js
+++ b/test/mysql/query-generator.test.js
@@ -389,6 +389,9 @@ if (Support.dialectIsMySQL()) {
        }, {
          arguments: ['myTable', [{name: "foo", value: true}, {name: 'bar', value: false}]],
          expectation: "INSERT INTO `myTable` (`name`,`value`) VALUES ('foo',true),('bar',false);"
+        }, {
+          arguments: ['myTable', [{name: 'foo'}, {name: 'bar'}], {ignoreDuplicates: true}],
+          expectation: "INSERT IGNORE INTO `myTable` (`name`) VALUES ('foo'),('bar');"
        }
      ],

--- a/test/sqlite/query-generator.test.js
+++ b/test/sqlite/query-generator.test.js
@@ -385,6 +385,9 @@ if (dialect === 'sqlite') {
          arguments: ['myTable', [{name: 'foo', foo: 1, nullValue: null}, {name: 'bar', foo: 2, nullValue: null}]],
          expectation: "INSERT INTO `myTable` (`name`,`foo`,`nullValue`) VALUES ('foo',1,NULL),('bar',2,NULL);",
          context: {options: {omitNull: true}} // Note: As above
+        }, {
+          arguments: ['myTable', [{name: 'foo'}, {name: 'bar'}], {ignoreDuplicates: true}],
+          expectation: "INSERT OR IGNORE INTO `myTable` (`name`) VALUES ('foo'),('bar');"
        }
      ],