Skip to content

Commit

Permalink
feat(product-import): Filter out failed products #116 (#121)
Browse files Browse the repository at this point in the history 
* feat(product-import): Filter out failed products #116

* chore(product-import): Refactor code - DRU #116
  • Loading branch information
@junajan
junajan authored Jun 6, 2018
1 parent 9707945 commit 63d296a
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 35 deletions.
52 changes: 30 additions & 22 deletions dist/product-import.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,25 @@ ProductImport = (function() {
return report;
};

ProductImport.prototype._filterOutProductsBySkus = function(products, blacklistSkus) {
return products.filter((function(_this) {
return function(product) {
var isProductBlacklisted, variantSkus, variants;
variants = product.variants.concat(product.masterVariant);
variantSkus = variants.map(function(v) {
return v.sku;
});
isProductBlacklisted = variantSkus.find(function(sku) {
return blacklistSkus.indexOf(sku) >= 0;
});
if (isProductBlacklisted) {
_this._summary.failed++;
}
return !isProductBlacklisted;
};
})(this));
};

ProductImport.prototype.performStream = function(chunk, cb) {
return this._processBatches(chunk).then(function() {
return cb();
Expand All @@ -197,41 +216,27 @@ ProductImport = (function() {
return _this._updateProductType(uniqueEnumUpdateActions);
}
}).then(function() {
var filteredProductsLength, originalLength, skus;
var filteredProductsLength, originalLength, reassignmentService;
originalLength = productsToProcess.length;
productsToProcess = productsToProcess.filter(_this._doesProductHaveSkus);
filteredProductsLength = originalLength - productsToProcess.length;
if (filteredProductsLength) {
_this.logger.warn("Filtering out " + filteredProductsLength + " product(s) which do not have SKU");
_this._summary.productsWithMissingSKU += filteredProductsLength;
}
skus = _this._extractUniqueSkus(productsToProcess);
if (skus.length) {
return _this._getExistingProductsForSkus(skus);
} else {
return [];
}
}).then(function(queriedEntries) {
var reassignmentService;
if (_this.variantReassignmentOptions.enabled) {
_this.logger.debug('execute reassignment process');
reassignmentService = new Reassignment(_this.client, _this.logger, _this.variantReassignmentOptions.retainExistingData);
return reassignmentService.execute(productsToProcess, _this._cache.productType).then(function(statistics) {
var skus;
_this._summary.variantReassignment = statistics;
if (statistics.processed > 0) {
skus = _this._extractUniqueSkus(productsToProcess);
if (skus.length) {
return _this._getExistingProductsForSkus(skus);
} else {
return queriedEntries;
}
} else {
return queriedEntries;
return reassignmentService.execute(productsToProcess, _this._cache.productType).then(function(res) {
_this._summary.variantReassignment = res.statistics;
if (res.failedSkus.length) {
_this.logger.warn("Removing " + res.failedSkus + " skus from processing due to a reassignment error");
productsToProcess = _this._filterOutProductsBySkus(productsToProcess, res.failedSkus);
}
return _this._getExistingProductsForSkus(_this._extractUniqueSkus(productsToProcess));
});
} else {
return queriedEntries;
return _this._getExistingProductsForSkus(_this._extractUniqueSkus(productsToProcess));
}
}).then(function(queriedEntries) {
if (_this.defaultAttributesService) {
Expand Down Expand Up @@ -271,6 +276,9 @@ ProductImport = (function() {
return new Promise((function(_this) {
return function(resolve, reject) {
var skuChunks;
if (skus.length === 0) {
return resolve([]);
}
skuChunks = _this.commonUtils._separateSkusChunksIntoSmallerChunks(skus, _this._getWhereQueryLimit());
return Promise.map(skuChunks, function(skus) {
var predicate;
Expand Down
44 changes: 31 additions & 13 deletions lib/product-import.coffee
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,21 @@ class ProductImport
}
report

_filterOutProductsBySkus: (products, blacklistSkus) ->
return products.filter (product) =>
variants = product.variants.concat(product.masterVariant)
variantSkus = variants.map (v) -> v.sku

# check if at least one SKU from product is in the blacklist
isProductBlacklisted = variantSkus.find (sku) ->
blacklistSkus.indexOf(sku) >= 0

if isProductBlacklisted
@_summary.failed++

# filter only products which are NOT blacklisted
not isProductBlacklisted

performStream: (chunk, cb) ->
@_processBatches(chunk).then -> cb()

Expand All @@ -135,24 +150,24 @@ class ProductImport
@logger.warn "Filtering out #{filteredProductsLength} product(s) which do not have SKU"
@_summary.productsWithMissingSKU += filteredProductsLength

skus = @_extractUniqueSkus(productsToProcess)
if skus.length then @_getExistingProductsForSkus(skus) else []
.then (queriedEntries) =>
if (@variantReassignmentOptions.enabled)
@logger.debug 'execute reassignment process'

reassignmentService = new Reassignment(@client, @logger,
@variantReassignmentOptions.retainExistingData)
@variantReassignmentOptions.retainExistingData)
reassignmentService.execute(productsToProcess, @_cache.productType)
.then((statistics) =>
@_summary.variantReassignment = statistics
if (statistics.processed > 0)
skus = @_extractUniqueSkus(productsToProcess)
if skus.length then @_getExistingProductsForSkus(skus) else queriedEntries
else
return queriedEntries
.then((res) =>
@_summary.variantReassignment = res.statistics

# if there are products which failed during reassignment, remove them from processing
if(res.failedSkus.length)
@logger.warn(
"Removing #{res.failedSkus} skus from processing due to a reassignment error"
)
productsToProcess = @_filterOutProductsBySkus(productsToProcess, res.failedSkus)
)
else
return queriedEntries
.then () =>
@_getExistingProductsForSkus(@_extractUniqueSkus(productsToProcess))
.then (queriedEntries) =>
if @defaultAttributesService
debug 'Ensuring default attributes'
Expand Down Expand Up @@ -185,6 +200,9 @@ class ProductImport

_getExistingProductsForSkus: (skus) =>
new Promise (resolve, reject) =>
if skus.length == 0
return resolve([])

skuChunks = @commonUtils._separateSkusChunksIntoSmallerChunks(
skus,
@_getWhereQueryLimit()
Expand Down
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@
"grunt-shell": "2.1.0",
"jasmine-node": "1.14.5",
"randomstring": "1.1.5",
"sinon": "^5.1.0",
"sphere-coffeelint": "git://github.com/sphereio/sphere-coffeelint.git#master"
},
"keywords": [
Expand Down
82 changes: 82 additions & 0 deletions test/integration/product-import.spec.coffee
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
os = require 'os'
path = require 'path'
sinon = require 'sinon'
debug = require('debug')('spec:it:sphere-product-import')
_ = require 'underscore'
_.mixin require 'underscore-mixins'
Expand Down Expand Up @@ -473,3 +474,84 @@ describe 'Product Importer integration tests', ->
done()
.catch (err) =>
done(_.prettify err)

it 'should handle product reassignment error', (done) ->
stubCreateOrUpdate = sinon.stub(@import, '_createOrUpdate')
.callThrough()

productDraft1 = createProduct()[0]
productDraft1.productType.id = @productType.id
productDraft1.categories[0].id = @category.id
productDraft2 = createProduct()[1]
productDraft2.productType.id = @productType.id
productDraft2.categories[0].id = @category.id
Promise.all([
ensureResource(@client.products, "masterData(staged(slug(en=\"#{productDraft1.slug.en}\")))", productDraft1)
ensureResource(@client.products, "masterData(staged(slug(en=\"#{productDraft2.slug.en}\")))", productDraft2)
])
.then () =>
productDraftChunk = {
productType:
typeId: 'product-type'
id: @productType.name
"name": {
"en": "reassigned-product"
},
"categories": [
{
"typeId": "category",
"id": "test-category"
}
],
"categoryOrderHints": {},
"slug": {
"en": "reassigned-product"
},
"masterVariant": {
"sku": "sku4",
"prices": [],
"images": [],
"attributes": [],
"assets": []
},
"variants": [
{
"sku": "sku3",
"prices": [
{
"value": {
"currencyCode": "GBP",
"centAmount": 'INVALID PRICE'
},
"id": "e9748681-e42f-45c4-b07f-48b92c6e910a"
}
],
"images": [],
"attributes": [],
"assets": []
}
],
"searchKeywords": {}
}
@import.performStream([productDraftChunk], Promise.resolve)
.then =>
expect(stubCreateOrUpdate.callCount).toEqual(1)
productsToProcess = stubCreateOrUpdate.firstCall.args[0]
queriedEntries = stubCreateOrUpdate.firstCall.args[1]

expect(productsToProcess.length).toBe(0)
expect(queriedEntries.length).toBe(0)
expect(@import._summary.failed).toBe(1)

expect(@import._summary.variantReassignment).toEqual({
anonymized: 0,
productTypeChanged: 0,
processed: 1,
succeeded: 0,
retries: 0,
errors: 1,
failedSkus: [ 'sku4', 'sku3' ]
})
done()
.catch (err) =>
done(_.prettify err)

0 comments on commit 63d296a

Please sign in to comment.