Merge pull request #1790 from sequelize/perf-groupJoinData

optimize groupJoinData to use a single loop

lib/dialects/abstract/query.js

@@ -339,90 +339,306 @@ module.exports = (function() {
         }
         }
       ]
       ]
   */
   */
+  /*
+   * Assumptions
+   * ID is not necessarily the first field
+   * All fields for a level is grouped in the same set (i.e. Panel.id, Task.id, Panel.title is not possible)
+   * Parent keys will be seen before any include/child keys
+   * Previous set won't necessarily be parent set (one parent could have two children, one child would then be previous set for the other)
+   */
 
 
-  // includeOptions are 'level'-specific where options is a general directive
-  var i = 0;
-  var groupJoinData = function(data, includeOptions, options) {
-    var results = []
-      , existingResult
-      , calleeData
-      , child
-      , calleeDataIgnore = ['__children']
-      , parseChildren = function(result) {
-        _.each(result.__children, function(children, key) {
-          result[key] = groupJoinData(children, (includeOptions.includeMap && includeOptions.includeMap[key]), options);
-        });
+   /*
+   * Author (MH) comment: This code is an unreadable mess, but its performant.
+   * groupJoinData is a performance critical function so we prioritize perf over readability.
+   */
 
 
-        delete result.__children;
+  var groupJoinData = function(rows, includeOptions, options) {
+    if (!rows.length) {
+      return [];
+    }
+
+    var
+      // Generic looping
+      i
+      , length
+      , $i
+      , $length
+      // Row specific looping
+      , rowsI
+      , rowsLength = rows.length
+      , row
+      // Key specific looping
+      , keys
+      , key
+      , keyI
+      , keyLength
+      , prevKey
+      , values
+      , topValues
+      , topExists
+      , previous
+      , checkExisting = options.checkExisting
+      // If we don't have to deduplicate we can pre-allocate the resulting array
+      , results = checkExisting ? [] : new Array(rowsLength)
+      , resultMap = {}
+      , includeMap = {}
+      , itemHash
+      , parentHash
+      , topHash
+      // Result variables for the respective functions
+      , $keyPrefix
+      , $keyPrefixString
+      , $prevKeyPrefixString
+      , $prevKeyPrefix
+      , $lastKeyPrefix
+      , $current
+      , $parent
+      // Map each key to an include option
+      , previousPiece
+      , buildIncludeMap = function (piece) {
+        if ($current.includeMap[piece]) {
+          includeMap[key] = $current = $current.includeMap[piece];
+          if (previousPiece) {
+            previousPiece = previousPiece+'.'+piece;
+          } else {            
+            previousPiece = piece;
+          }
+          includeMap[previousPiece] = $current;
+        }
+      }
+      // Calcuate the last item in the array prefix ('Results' for 'User.Results.id')
+      , lastKeyPrefixMemo = {}
+      , lastKeyPrefix = function (key) {
+        if (!lastKeyPrefixMemo[key]) {
+          var prefix = keyPrefix(key)
+            , length = prefix.length;
+
+          lastKeyPrefixMemo[key] = !length ? '' : prefix[length - 1];
+        }
+        return lastKeyPrefixMemo[key];
+      }
+      // Calculate the string prefix of a key ('User.Results' for 'User.Results.id')
+      , keyPrefixStringMemo = {}
+      , keyPrefixString = function (key, memo) {
+        if (!memo[key]) {
+          memo[key] = key.substr(0, key.lastIndexOf('.'));
+        }
+        return memo[key];
       }
       }
-      , primaryKeyAttribute
-      , primaryKeyMap = {};
+      // Removes the prefix from a key ('id' for 'User.Results.id')
+      , removeKeyPrefixMemo = {}
+      , removeKeyPrefix = function (key) {
+        if (!removeKeyPrefixMemo[key]) {
+          var index = key.lastIndexOf('.');
+          removeKeyPrefixMemo[key] = key.substr(index === -1 ? 0 : index + 1);
+        }
+        return removeKeyPrefixMemo[key];
+      }
+      // Calculates the array prefix of a key (['User', 'Results'] for 'User.Results.id')
+      , keyPrefixMemo = {}
+      , keyPrefix = function (key) {
+        // We use a double memo and keyPrefixString so that different keys with the same prefix will receive the same array instead of differnet arrays with equal values
+        if (!keyPrefixMemo[key]) {
+          var prefixString = keyPrefixString(key, keyPrefixStringMemo);
+          if (!keyPrefixMemo[prefixString]) {
+            keyPrefixMemo[prefixString] = prefixString ? prefixString.split(".") : [];
+          }
+          keyPrefixMemo[key] = keyPrefixMemo[prefixString];
+        }
+        return keyPrefixMemo[key];
+      }
+      , primaryKeyAttributes
+      , prefix;
 
 
-    // Ignore all include keys on main data
-    if (includeOptions.includeNames) {
-      calleeDataIgnore = calleeDataIgnore.concat(includeOptions.includeNames);
-    }
-    if (includeOptions.model.primaryKeyAttributes.length === 1) {
-      primaryKeyAttribute = includeOptions.model.primaryKeyAttribute;
-    }
+    for (rowsI = 0; rowsI < rowsLength; rowsI++) {
+      row = rows[rowsI];
+
+      // Keys are the same for all rows, so only need to compute them on the first row
+      if (rowsI === 0) {
+        keys = Object.keys(row);
+        keyLength = keys.length;
+      }
 
 
-    data.forEach(function parseRow(row) {
-      row = Dot.transform(row);
-      calleeData = row;
+      if (checkExisting) {
+        topExists = false;
 
 
-      // If there are :M associations included we need to see if the main result of the row has already been identified
-      if (options.checkExisting) {
-        if (primaryKeyAttribute) {
-          // If we can, detect equality on the singular primary key
-          existingResult = primaryKeyMap[calleeData[primaryKeyAttribute]];
+        // Compute top level hash key (this is usually just the primary key values)
+        $length = includeOptions.model.primaryKeyAttributes.length;
+        if ($length === 1) {
+          topHash = row[includeOptions.model.primaryKeyAttributes[0]];
         } else {
         } else {
-          // If we can't identify on a singular primary key, do a full row equality check
-          existingResult = _.find(results, function(result) {
-            return Utils._.isEqual(_.omit(result, calleeDataIgnore), calleeData);
-          });
+          topHash = '';
+          for ($i = 0; $i < $length; $i++) {
+            topHash += row[includeOptions.model.primaryKeyAttributes[$i]];
+          }
         }
         }
-      } else {
-        existingResult = null;
       }
       }
 
 
-      if (!existingResult) {
-        results.push(existingResult = calleeData);
-        if (options.checkExisting && primaryKeyAttribute) {
-          primaryKeyMap[existingResult[primaryKeyAttribute]] = existingResult;
+      topValues = values = {};
+      $prevKeyPrefix = undefined;
+      for (keyI = 0; keyI < keyLength; keyI++) {
+        key = keys[keyI];
+
+        // The string prefix isn't actualy needed
+        // We use it so keyPrefix for different keys will resolve to the same array if they have the same prefix
+        // TODO: Find a better way?
+        $keyPrefixString = keyPrefixString(key, keyPrefixStringMemo);
+        $keyPrefix = keyPrefix(key);
+
+        // On the first row we compute the includeMap
+        if (rowsI === 0 && includeMap[key] === undefined) {
+          if (!$keyPrefix.length) {
+            includeMap[key] = includeMap[''] = includeOptions;
+          } else {
+            $current = includeOptions;
+            previousPiece = undefined;
+            $keyPrefix.forEach(buildIncludeMap);
+          }
         }
         }
-      }
 
 
-      for (var attrName in row) {
-        if (row.hasOwnProperty(attrName)) {
-          // Child if object, and is an child include
-          child = Object(row[attrName]) === row[attrName] && includeOptions.includeMap && includeOptions.includeMap[attrName];
+        // End of key set
+        if ($prevKeyPrefix !== undefined && $prevKeyPrefix !== $keyPrefix) {
+          if (checkExisting) {
+            // Compute hash key for this set instance
+            // TODO: Optimize
+            length = $prevKeyPrefix.length;
+            $parent = null;
+            parentHash = null;
+
+            if (length) {
+              for (i = 0; i < length; i++) {
+                prefix = $parent ? $parent+'.'+$prevKeyPrefix[i] : $prevKeyPrefix[i];
+                primaryKeyAttributes = includeMap[prefix].model.primaryKeyAttributes;
+                $length = primaryKeyAttributes.length;
+                if ($length === 1) {
+                  itemHash = prefix+row[prefix+'.'+primaryKeyAttributes[0]];
+                } else {
+                  itemHash = prefix;
+                  for ($i = 0; $i < $length; $i++) {
+                    itemHash += row[prefix+'.'+primaryKeyAttributes[$i]];
+                  }
+                }
+                if (!parentHash) {
+                  parentHash = topHash;
+                }
+
+                itemHash = parentHash + itemHash;
+                $parent = prefix;
+                if (i < length - 1) {
+                  parentHash = itemHash;
+                }
+              }
+            } else {
+              itemHash = topHash;
+            }
 
 
-          if (child) {
-            // Make sure nested object is available
-            if (!existingResult.__children) {
-              existingResult.__children = {};
+            if (itemHash === topHash) {
+              if (!resultMap[itemHash]) {
+                resultMap[itemHash] = values;
+              } else {
+                topExists = true;
+              }
+            } else {
+              if (!resultMap[itemHash]) {
+                $parent = resultMap[parentHash];
+                $lastKeyPrefix = lastKeyPrefix(prevKey);
+                //console.log($parent, prevKey, $lastKeyPrefix);
+                if (includeMap[prevKey].association.isSingleAssociation) {
+                  $parent[$lastKeyPrefix] = resultMap[itemHash] = values;
+                } else {
+                  if (!$parent[$lastKeyPrefix]) {
+                    $parent[$lastKeyPrefix] = [];
+                  }
+                  $parent[$lastKeyPrefix].push(resultMap[itemHash] = values);
+                }
+              }
             }
             }
-            if (!existingResult.__children[attrName]) {
-              existingResult.__children[attrName] = [];
+
+            // Reset values
+            values = {};
+          } else {
+            // If checkExisting is false it's because there's only 1:1 associations in this query
+            // However we still need to map onto the appropriate parent
+            // For 1:1 we map forward, initializing the value object on the parent to be filled in the next iterations of the loop
+            $current = topValues;
+            length = $keyPrefix.length;
+            if (length) {
+              for (i = 0; i < length; i++) {
+                if (i === length -1) {
+                  values = $current[$keyPrefix[i]] = {};
+                }
+                $current = $current[$keyPrefix[i]];
+              }
             }
             }
+          }
+        }
 
 
-            existingResult.__children[attrName].push(row[attrName]);
+        // End of iteration, set value and set prev values (for next iteration)
+        values[removeKeyPrefix(key)] = row[key];
+        prevKey = key;
+        $prevKeyPrefix = $keyPrefix;
+        $prevKeyPrefixString = $keyPrefixString;
+      }
 
 
-            // Remove from main
-            delete existingResult[attrName];
+      if (checkExisting) {
+        length = $prevKeyPrefix.length;
+        $parent = null;
+        parentHash = null;
+
+        if (length) {
+          for (i = 0; i < length; i++) {
+            prefix = $parent ? $parent+'.'+$prevKeyPrefix[i] : $prevKeyPrefix[i];
+            primaryKeyAttributes = includeMap[prefix].model.primaryKeyAttributes;
+            $length = primaryKeyAttributes.length;
+            if ($length === 1) {
+              itemHash = prefix+row[prefix+'.'+primaryKeyAttributes[0]];
+            } else {
+              itemHash = prefix;
+              for ($i = 0; $i < $length; $i++) {
+                itemHash += row[prefix+'.'+primaryKeyAttributes[$i]];
+              }
+            }
+            if (!parentHash) {
+              parentHash = topHash;
+            }
+
+            itemHash = parentHash + itemHash;
+            $parent = prefix;
+            if (i < length - 1) {
+              parentHash = itemHash;
+            }
           }
           }
+        } else {
+          itemHash = topHash;
         }
         }
-      }
 
 
-      // parseChildren in same loop if no duplicate values are possible
-      if (!options.checkExisting) {
-        parseChildren(existingResult);
+        if (itemHash === topHash) {
+          if (!resultMap[itemHash]) {
+            resultMap[itemHash] = values;
+          } else {
+            topExists = true;
+          }
+        } else {
+          if (!resultMap[itemHash]) {
+            $parent = resultMap[parentHash];
+            $lastKeyPrefix = lastKeyPrefix(prevKey);
+            //console.log($parent, prevKey, $lastKeyPrefix);
+            if (includeMap[prevKey].association.isSingleAssociation) {
+              $parent[$lastKeyPrefix] = resultMap[itemHash] = values;
+            } else {
+              if (!$parent[$lastKeyPrefix]) {
+                $parent[$lastKeyPrefix] = [];
+              }
+              $parent[$lastKeyPrefix].push(resultMap[itemHash] = values);
+            }
+          }
+        }
+        if (!topExists) {
+          results.push(topValues);
+        }
+      } else {
+        results[rowsI] = topValues;
       }
       }
-    });
-
-    // parseChildren after row parsing if duplicate values are possible
-    if (options.checkExisting) {
-      results.forEach(parseChildren);
     }
     }
 
 
     return results;
     return results;

lib/instance.js

@@ -362,8 +362,8 @@ module.exports = (function() {
       , self = this
       , self = this
       , accessor = Utils._.camelize(key)
       , accessor = Utils._.camelize(key)
       , childOptions
       , childOptions
-      , primaryKeyAttribute = include.model.primaryKeyAttribute
-      , isEmpty = value[0] && value[0][primaryKeyAttribute] === null;
+      , primaryKeyAttribute  = include.model.primaryKeyAttribute
+      , isEmpty;
 
 
     if (!isEmpty) {
     if (!isEmpty) {
       childOptions = {
       childOptions = {
@@ -382,9 +382,15 @@ module.exports = (function() {
       accessor = accessor.slice(0, 1).toLowerCase() + accessor.slice(1);
       accessor = accessor.slice(0, 1).toLowerCase() + accessor.slice(1);
 
 
       if (association.isSingleAssociation) {
       if (association.isSingleAssociation) {
+        if (Array.isArray(value)) {
+          value = value[0];
+        }
+
+        isEmpty = value && value[primaryKeyAttribute] === null;
         accessor = Utils.singularize(accessor, self.Model.options.language);
         accessor = Utils.singularize(accessor, self.Model.options.language);
-        self[accessor] = self.dataValues[accessor] = isEmpty ? null : include.model.build(value[0], childOptions);
+        self[accessor] = self.dataValues[accessor] = isEmpty ? null : include.model.build(value, childOptions);
       } else {
       } else {
+        isEmpty = value[0] && value[0][primaryKeyAttribute] === null;
         self[accessor] = self.dataValues[accessor] = isEmpty ? [] : include.model.bulkBuild(value, childOptions);
         self[accessor] = self.dataValues[accessor] = isEmpty ? [] : include.model.bulkBuild(value, childOptions);
       }
       }
     }
     }