Merge pull request #127 from sphereio/116-variant-reassignment

feat(product-import): Variant reassignment

lib/price-import.coffee

@@ -52,6 +52,22 @@ class PriceImport extends ProductImport
             Promise.resolve(@_summary)
     ,{concurrency: 1}
 
+  _handleProcessResponse: (res) =>
+    if res.isFulfilled()
+      @_handleFulfilledResponse(res)
+    else if res.isRejected()
+      error = serializeError res.reason()
+
+      @_summary.failed++
+      if @errorDir
+        errorFile = path.join(@errorDir, "error-#{@_summary.failed}.json")
+        fs.outputJsonSync(errorFile, error, {spaces: 2})
+
+      if _.isFunction(@errorCallback)
+        @errorCallback(error, @logger)
+      else
+        @logger.error "Error callback has to be a function!"
+
   _handleFulfilledResponse: (r) =>
     switch r.value().statusCode
       when 201 then @_summary.created++

lib/product-import.coffee

@@ -13,6 +13,7 @@ UnknownAttributesFilter = require './unknown-attributes-filter'
 CommonUtils = require './common-utils'
 EnsureDefaultAttributes = require './ensure-default-attributes'
 util = require 'util'
+Reassignment = require('commercetools-node-variant-reassignment').default
 
 class ProductImport
 
@@ -45,6 +46,12 @@ class ProductImport
     # possible values:
     # always, publishedOnly, stagedAndPublishedOnly
     @publishingStrategy = options.publishingStrategy or false
+    @variantReassignmentOptions = options.variantReassignmentOptions or {}
+    if @variantReassignmentOptions.enabled
+      @reassignmentService = new Reassignment(@client, @logger,
+        (error) => @_handleErrorResponse(error),
+        @variantReassignmentOptions.retainExistingData)
+
     @_configErrorHandling(options)
     @_resetCache()
     @_resetSummary()
@@ -88,6 +95,9 @@ class ProductImport
       productTypeUpdated: 0
       errorDir: @errorDir
     if @filterUnknownAttributes then @_summary.unknownAttributeNames = []
+    if @variantReassignmentOptions.enabled
+      @_summary.variantReassignment = null
+      @reassignmentService._resetStats()
 
   summaryReport: (filename) ->
     message = "Summary: there were #{@_summary.created + @_summary.updated} imported products " +
@@ -106,6 +116,18 @@ class ProductImport
     }
     report
 
+  _filterOutProductsBySkus: (products, blacklistSkus) ->
+    return products.filter (product) =>
+      variants = product.variants.concat(product.masterVariant)
+      variantSkus = variants.map (v) -> v.sku
+
+      # check if at least one SKU from product is in the blacklist
+      isProductBlacklisted = variantSkus.find (sku) ->
+        blacklistSkus.indexOf(sku) >= 0
+
+      # filter only products which are NOT blacklisted
+      not isProductBlacklisted
+
   performStream: (chunk, cb) ->
     @_processBatches(chunk).then -> cb()
 
@@ -132,8 +154,20 @@ class ProductImport
           @logger.warn "Filtering out #{filteredProductsLength} product(s) which do not have SKU"
           @_summary.productsWithMissingSKU += filteredProductsLength
 
-        skus = @_extractUniqueSkus(productsToProcess)
-        if skus.length then @_getExistingProductsForSkus(skus) else []
+        if (@variantReassignmentOptions.enabled)
+          @logger.debug 'execute reassignment process'
+
+          @reassignmentService.execute(productsToProcess, @_cache.productType)
+          .then((res) =>
+            # if there are products which failed during reassignment, remove them from processing
+            if(res.badRequestSKUs.length)
+              @logger.warn(
+                "Removing #{res.badRequestSKUs} skus from processing due to a reassignment error"
+              )
+              productsToProcess = @_filterOutProductsBySkus(productsToProcess, res.badRequestSKUs)
+          )
+      .then () =>
+        @_getExistingProductsForSkus(@_extractUniqueSkus(productsToProcess))
       .then (queriedEntries) =>
         if @defaultAttributesService
           debug 'Ensuring default attributes'
@@ -146,9 +180,16 @@ class ProductImport
         @_createOrUpdate productsToProcess, queriedEntries
       .then (results) =>
         _.each results, (r) =>
-          @_handleProcessResponse(r)
+          if r.isFulfilled()
+            @_handleFulfilledResponse(r)
+          else if r.isRejected()
+            @_handleErrorResponse(r.reason())
+
         Promise.resolve(@_summary)
     , { concurrency: 1 } # run 1 batch at a time
+    .then =>
+      if @variantReassignmentOptions.enabled
+        @_summary.variantReassignment = @reassignmentService.statistics
 
   _getWhereQueryLimit: ->
     client = @client.productProjections
@@ -166,6 +207,9 @@ class ProductImport
 
   _getExistingProductsForSkus: (skus) =>
     new Promise (resolve, reject) =>
+      if skus.length == 0
+        return resolve([])
+
       skuChunks = @commonUtils._separateSkusChunksIntoSmallerChunks(
         skus,
         @_getWhereQueryLimit()
@@ -194,21 +238,18 @@ class ProductImport
         Error not logged as error limit of #{@errorLimit} has reached.
       "
 
-  _handleProcessResponse: (res) =>
-    if res.isFulfilled()
-      @_handleFulfilledResponse(res)
-    else if res.isRejected()
-      error = serializeError res.reason()
+  _handleErrorResponse: (error) =>
+    error = serializeError error
 
-      @_summary.failed++
-      if @errorDir
-        errorFile = path.join(@errorDir, "error-#{@_summary.failed}.json")
-        fs.outputJsonSync(errorFile, error, {spaces: 2})
+    @_summary.failed++
+    if @errorDir
+      errorFile = path.join(@errorDir, "error-#{@_summary.failed}.json")
+      fs.outputJsonSync(errorFile, error, {spaces: 2})
 
-      if _.isFunction(@errorCallback)
-        @errorCallback(error, @logger)
-      else
-        @logger.error "Error callback has to be a function!"
+    if _.isFunction(@errorCallback)
+      @errorCallback(error, @logger)
+    else
+      @logger.error "Error callback has to be a function!"
 
   _handleFulfilledResponse: (res) =>
     switch res.value().statusCode

package.json

@@ -39,6 +39,7 @@
   },
   "dependencies": {
     "bluebird": "^3.0.0",
+    "commercetools-node-variant-reassignment": "1.1.0",
     "cuid": "^1.3.8",
     "debug": "2.6.8",
     "fs-extra": "3.0.1",
@@ -62,6 +63,7 @@
     "grunt-shell": "2.1.0",
     "jasmine-node": "1.14.5",
     "randomstring": "1.1.5",
+    "sinon": "^5.1.0",
     "sphere-coffeelint": "git://github.com/sphereio/sphere-coffeelint.git#master"
   },
   "keywords": [

readme/product-import.md

@@ -39,6 +39,8 @@ Note that this tool can import only products which have at least one variant wit
         * the product that should be updated
         * the product that is being imported
       * an _array_ of update actions that should be ignored
+    * variantReassignmentOptions
+      * enabled: when set to `true`, [reassignment module](https://github.com/commercetools/commercetools-node-variant-reassignment) will run before product import.
 
 #### Sample configuration object for cli:
 

test/integration/product-import.spec.coffee

@@ -1,5 +1,6 @@
 os = require 'os'
 path = require 'path'
+sinon = require 'sinon'
 debug = require('debug')('spec:it:sphere-product-import')
 _ = require 'underscore'
 _.mixin require 'underscore-mixins'
@@ -384,3 +385,194 @@ describe 'Product Importer integration tests', ->
         expect(product.masterVariant.attributes.length).toBe(900)
         expect(product.variants[0].attributes.length).toBe(635)
         done()
+
+  describe 'product variant reassignment', ->
+    beforeEach (done) ->
+      reassignmentConfig = _.extend config, { variantReassignmentOptions: { enabled: true } }
+      @import = new ProductImport logger, reassignmentConfig
+      done()
+
+#   Reassignment before:
+#   existing product "foo": sku1, sku2, sku3
+#   existing product "no-category": sku4, sku5, sku6
+#   new product draft "reassigned-product": sku4, sku3
+#   ---
+#   Reassignment before:
+#   "no-category" will be reassigned because of matched by master variant "reassinged-product": sku4, sku3
+#   1 new anonymized product "no-category-randomSlug": sku5, sku6
+#   existing product "foo": sku1, sku2
+    it 'should reassign product variants', (done) ->
+      productDraft1 = createProduct()[0]
+      productDraft1.productType.id = @productType.id
+      productDraft1.categories[0].id = @category.id
+      productDraft2 = createProduct()[1]
+      productDraft2.productType.id = @productType.id
+      productDraft2.categories[0].id = @category.id
+      Promise.all([
+        ensureResource(@client.products, "masterData(staged(slug(en=\"#{productDraft1.slug.en}\")))", productDraft1)
+        ensureResource(@client.products, "masterData(staged(slug(en=\"#{productDraft2.slug.en}\")))", productDraft2)
+      ])
+      .then () =>
+        productDraftChunk = {
+          productType:
+            typeId: 'product-type'
+            id: @productType.name
+          "name": {
+            "en": "reassigned-product"
+          },
+          "categories": [
+            {
+              "typeId": "category",
+              "id": "test-category"
+            }
+          ],
+          "categoryOrderHints": {},
+          "slug": {
+            "en": "reassigned-product"
+          },
+          "masterVariant": {
+            "sku": "sku4",
+            "prices": [],
+            "images": [],
+            "attributes": [],
+            "assets": []
+          },
+          "variants": [
+            {
+              "sku": "sku3",
+              "prices": [
+                {
+                  "value": {
+                    "currencyCode": "GBP",
+                    "centAmount": 9
+                  },
+                  "id": "e9748681-e42f-45c4-b07f-48b92c6e910a"
+                }
+              ],
+              "images": [],
+              "attributes": [],
+              "assets": []
+            }
+          ],
+          "searchKeywords": {}
+        }
+        @import.performStream([productDraftChunk], Promise.resolve)
+      .then =>
+        expect(@import._summary.variantReassignment.anonymized).toEqual(1)
+        expect(@import._summary.variantReassignment.productTypeChanged).toEqual(0)
+        expect(@import._summary.variantReassignment.processed).toEqual(1)
+        expect(@import._summary.variantReassignment.succeeded).toEqual(1)
+        expect(@import._summary.variantReassignment.transactionRetries).toEqual(0)
+        expect(@import._summary.variantReassignment.badRequestErrors).toEqual(0)
+        expect(@import._summary.variantReassignment.processedSkus).toEqual([ 'sku4', 'sku3' ])
+        expect(@import._summary.variantReassignment.badRequestSKUs).toEqual([])
+        expect(@import._summary.variantReassignment.anonymizedSlug[0].indexOf('no-category')).not.toEqual(-1)
+
+        @fetchProducts(@productType.id)
+      .then ({ body: { results } }) =>
+        expect(results.length).toEqual(3)
+        reassignedProductNoCategory = _.find results, (p) => p.name.en == 'reassigned-product'
+        expect(reassignedProductNoCategory.slug.en).toEqual('reassigned-product')
+        expect(reassignedProductNoCategory.masterVariant.sku).toEqual('sku4')
+        expect(reassignedProductNoCategory.variants.length).toEqual(1)
+        expect(reassignedProductNoCategory.variants[0].sku).toEqual('sku3')
+
+        fooProduct = _.find results, (p) => p.name.en == 'foo'
+        expect(fooProduct.slug.en).toEqual('foo')
+        expect(fooProduct.masterVariant.sku).toEqual('sku1')
+        expect(fooProduct.variants.length).toEqual(1)
+        expect(fooProduct.variants[0].sku).toEqual('sku2')
+
+        anonymizedProduct = _.find results, (p) => p.name.en == 'no-category'
+        expect(anonymizedProduct.slug.ctsd).toBeDefined()
+        expect(anonymizedProduct.variants.length).toEqual(1)
+        skus = anonymizedProduct.variants.concat(anonymizedProduct.masterVariant)
+          .map((v) => v.sku)
+        expect(skus).toContain('sku5')
+        expect(skus).toContain('sku6')
+        done()
+      .catch (err) =>
+        done(_.prettify err)
+
+    it 'should handle product reassignment error', (done) ->
+      stubCreateOrUpdate = sinon.stub(@import, '_createOrUpdate')
+        .callThrough()
+
+      productDraft1 = createProduct()[0]
+      productDraft1.productType.id = @productType.id
+      productDraft1.categories[0].id = @category.id
+      productDraft2 = createProduct()[1]
+      productDraft2.productType.id = @productType.id
+      productDraft2.categories[0].id = @category.id
+      Promise.all([
+        ensureResource(@client.products, "masterData(staged(slug(en=\"#{productDraft1.slug.en}\")))", productDraft1)
+        ensureResource(@client.products, "masterData(staged(slug(en=\"#{productDraft2.slug.en}\")))", productDraft2)
+      ])
+      .then () =>
+        productDraftChunk = {
+          productType:
+            typeId: 'product-type'
+            id: @productType.name
+          "name": {
+            "en": "reassigned-product"
+          },
+          "categories": [
+            {
+              "typeId": "category",
+              "id": "test-category"
+            }
+          ],
+          "categoryOrderHints": {},
+          "slug": {
+            "en": "reassigned-product"
+          },
+          "masterVariant": {
+            "sku": "sku4",
+            "prices": [],
+            "images": [],
+            "attributes": [],
+            "assets": []
+          },
+          "variants": [
+            {
+              "sku": "sku3",
+              "prices": [
+                {
+                  "value": {
+                    "currencyCode": "GBP",
+                    "centAmount": 'INVALID PRICE'
+                  },
+                  "id": "e9748681-e42f-45c4-b07f-48b92c6e910a"
+                }
+              ],
+              "images": [],
+              "attributes": [],
+              "assets": []
+            }
+          ],
+          "searchKeywords": {}
+        }
+        @import.performStream([productDraftChunk], Promise.resolve)
+      .then =>
+        expect(stubCreateOrUpdate.callCount).toEqual(1)
+        productsToProcess = stubCreateOrUpdate.firstCall.args[0]
+        queriedEntries = stubCreateOrUpdate.firstCall.args[1]
+
+        expect(productsToProcess.length).toBe(0)
+        expect(queriedEntries.length).toBe(0)
+        expect(@import._summary.failed).toBe(1)
+
+        expect(@import._summary.variantReassignment).toEqual({
+          anonymized: 0,
+          productTypeChanged: 0,
+          processed: 1,
+          succeeded: 0,
+          transactionRetries: 0,
+          badRequestErrors: 1,
+          processedSkus: ['sku4', 'sku3']
+          badRequestSKUs: ['sku4', 'sku3']
+          anonymizedSlug: []
+        })
+        done()
+      .catch (err) =>
+        done(_.prettify err)