[Sendgrid lists] - multistatus responses (#2664)

* saving progress - nothing working

* saving progress

* saving progress

* refactoring

* updates

* fixing test

* fixing test

* fixing a single unit test

* minor refactor

* tests passing

* test cange after Sayan review

* removing retry in upsert function

* removing unused field

packages/destination-actions/src/destinations/sendgrid-audiences/syncAudience/__tests__/index.test.ts

@@ -1,5 +1,5 @@
 import nock from 'nock'
-import { createTestEvent, createTestIntegration, SegmentEvent, PayloadValidationError } from '@segment/actions-core'
+import { createTestEvent, createTestIntegration, SegmentEvent, IntegrationError, ErrorCodes } from '@segment/actions-core'
 import Definition from '../../index'
 import { Settings } from '../../generated-types'
 import { validatePhone, toDateFormat } from '../utils'
@@ -122,15 +122,13 @@ describe('SendgridAudiences.syncAudience', () => {
         first_name: 'fname', 
         last_name: 'lname', 
         street: '123 Main St',
-        address_line_2: 123456, // should be stringified
+        address_line_2: "address line 2", 
         city: 'SF', 
         state: 'CA',
         country: 'US',
         postal_code: "N88EU",
         custom_text_fields: {
-          custom_field_1: 'custom_field_1_value',
-          custom_field_4: false, // should be removed 
-          custom_field_5: null // should be removed
+          custom_field_1: 'custom_field_1_value'
         },
         custom_number_fields: {
           custom_field_2: 2345
@@ -152,7 +150,7 @@ describe('SendgridAudiences.syncAudience', () => {
           first_name: 'fname',
           last_name: 'lname',
           address_line_1: '123 Main St',
-          address_line_2: '123456',
+          address_line_2: 'address line 2',
           city: 'SF', 
           state_province_region: 'CA',
           country: 'US',
@@ -231,14 +229,14 @@ describe('SendgridAudiences.syncAudience', () => {
     }
 
     nock('https://api.sendgrid.com').put('/v3/marketing/contacts', upsertAddBatchExpectedPayload).reply(200, {})
-    const responses = await testDestination.testBatchAction('syncAudience', {
+    const responses = await testDestination.executeBatch('syncAudience', {
       events,
       settings,
-      useDefaultMappings: true,
       mapping
     })
-    expect(responses.length).toBe(1)
+    expect(responses.length).toBe(2)
     expect(responses[0].status).toBe(200)
+    expect(responses[1].status).toBe(200)
   })
 
   it('should remove a single Contact from a Sendgrid list correctly', async () => {
@@ -320,17 +318,15 @@ describe('SendgridAudiences.syncAudience', () => {
 
     nock('https://api.sendgrid.com').delete(deletePath).reply(200, {})
 
-    const responses = await testDestination.testBatchAction('syncAudience', {
+    const responses = await testDestination.executeBatch('syncAudience', {
       events,
       settings,
-      useDefaultMappings: true,
       mapping
     })
 
-    expect(responses.length).toBe(3)
+    expect(responses.length).toBe(2)
     expect(responses[0].status).toBe(200)
     expect(responses[1].status).toBe(200)
-    expect(responses[2].status).toBe(200)
   })
 
   it('should add and remove multiple Contacts from a Sendgrid list correctly', async () => {
@@ -394,10 +390,9 @@ describe('SendgridAudiences.syncAudience', () => {
 
     nock('https://api.sendgrid.com').delete(deletePath).reply(200, {})
 
-    const responses = await testDestination.testBatchAction('syncAudience', {
+    const responses = await testDestination.executeBatch('syncAudience', {
       events,
       settings,
-      useDefaultMappings: true,
       mapping
     })
 
@@ -408,7 +403,7 @@ describe('SendgridAudiences.syncAudience', () => {
     expect(responses[3].status).toBe(200)
   })
 
-  it('should retry upserting to add Contacts after Sendgrid initially rejects some emails', async () => {
+  it('should throw 429s if batch contains at least one email which SendGrid rejects as invalid', async () => {
     const badPayload = JSON.parse(JSON.stringify(addPayload))
     badPayload.traits.email = '[email protected]'
     delete badPayload.traits?.external_id
@@ -454,19 +449,34 @@ describe('SendgridAudiences.syncAudience', () => {
       ]
     }
 
+    const multiStatusRespError1 = {
+      "status": 429,
+      "errortype": "RETRYABLE_ERROR",
+      "errormessage": "Batch payload rejected by SendGrid due to at least one invalid email. Batch payload will be retried without the invalid email(s).",
+      "errorreporter": "INTEGRATIONS"
+    }
+
+    const multiStatusRespError2 = {
+      "status": 400,
+      "errortype": "PAYLOAD_VALIDATION_FAILED",
+      "errormessage": "SendGrid rejected email address [email protected] as invalid",
+      "errorreporter": "INTEGRATIONS"
+    }
+
     nock('https://api.sendgrid.com').put('/v3/marketing/contacts', upsertAddBatchExpectedPayload).reply(400, responseError)
     nock('https://api.sendgrid.com').put('/v3/marketing/contacts', addBatchExpectedPayload2).reply(200, {})
 
-    const responses = await testDestination.testBatchAction('syncAudience', {
+    const responses = await testDestination.executeBatch('syncAudience', {
       events,
       settings,
-      useDefaultMappings: true,
       mapping
     })
 
     expect(responses.length).toBe(2)
-    expect(responses[0].status).toBe(400)
-    expect(responses[1].status).toBe(200)
+    expect(responses[0].status).toBe(429)
+    expect(responses[1].status).toBe(400)
+    expect(responses[0]).toMatchObject(multiStatusRespError1)
+    expect(responses[1]).toMatchObject(multiStatusRespError2)
   })
 
   it('should throw an error if a non batch payload is missing identifiers', async () => {
@@ -485,11 +495,11 @@ describe('SendgridAudiences.syncAudience', () => {
         mapping
       })
     ).rejects.toThrowError(
-      new PayloadValidationError(`No valid payloads found`)
+      new IntegrationError(`At least one identifier from Email Address, Phone Number ID, Anonymous ID or External ID is required.`, ErrorCodes.PAYLOAD_VALIDATION_FAILED, 400)
     )
   })
 
-  it('should throw an error if a all payloads in a batch are invalid', async () => {
+  it('should return multistatus response with all errors if all payloads in a batch are invalid', async () => {
     const badPayload = JSON.parse(JSON.stringify(addPayload))
     delete badPayload.traits?.email
     delete badPayload.traits?.external_id
@@ -498,16 +508,24 @@ describe('SendgridAudiences.syncAudience', () => {
 
     const badPayload2 = JSON.parse(JSON.stringify(badPayload))
 
+    const responseError = {
+      status: 400,
+      errortype: 'PAYLOAD_VALIDATION_FAILED',
+      errormessage: 'At least one identifier from Email Address, Phone Number ID, Anonymous ID or External ID is required.',
+      errorreporter: 'INTEGRATIONS'
+    }
+
     const events = [createTestEvent(badPayload), createTestEvent(badPayload2)]
 
-    await expect(
-      testDestination.testBatchAction('syncAudience', {
-        events,
-        settings,
-        useDefaultMappings: true,
-        mapping
-      })
-    ).rejects.toThrowError(new PayloadValidationError(`No valid payloads found`))
+    const responses = await testDestination.executeBatch('syncAudience', {
+      events,
+      settings,
+      mapping
+    })
+
+    expect(responses.length).toBe(2)
+    expect(responses[0]).toMatchObject(responseError)
+    expect(responses[1]).toMatchObject(responseError)
   })
 
   it('should do multiple search and a single remove request for large batch with many identifiers but less than 100 contacts', async () => {
@@ -557,19 +575,16 @@ describe('SendgridAudiences.syncAudience', () => {
 
     nock('https://api.sendgrid.com').delete(deletePath).reply(200, {})
 
-    const responses = await testDestination.testBatchAction('syncAudience', {
+    const responses = await testDestination.executeBatch('syncAudience', {
       events,
       settings,
-      useDefaultMappings: true,
       mapping
     })
 
-    expect(responses.length).toBe(5)
-    expect(responses[0].status).toBe(200)
-    expect(responses[1].status).toBe(200)
-    expect(responses[2].status).toBe(200)
-    expect(responses[3].status).toBe(200)
-    expect(responses[4].status).toBe(200)
+    expect(responses.length).toBe(30)
+    for(let i = 0; i < 30; i++) {
+      expect(responses[i].status).toBe(200)
+    }
   })
 
   it('should do multiple search and a multiple remove requests for large batch with many identifiers and more than 100 contacts', async () => {
@@ -624,20 +639,16 @@ describe('SendgridAudiences.syncAudience', () => {
     nock('https://api.sendgrid.com').delete(deletePath).reply(200, {})
     nock('https://api.sendgrid.com').delete(deletePath2).reply(200, {})
 
-    const responses = await testDestination.testBatchAction('syncAudience', {
+    const responses = await testDestination.executeBatch('syncAudience', {
       events,
       settings,
-      useDefaultMappings: true,
       mapping
     })
 
-    expect(responses.length).toBe(6)
-    expect(responses[0].status).toBe(200)
-    expect(responses[1].status).toBe(200)
-    expect(responses[2].status).toBe(200)
-    expect(responses[3].status).toBe(200)
-    expect(responses[4].status).toBe(200)
-    expect(responses[5].status).toBe(200)
+    expect(responses.length).toBe(120)
+    for(let i = 0; i < 120; i++) {
+      expect(responses[i].status).toBe(200)
+    }
   })
 
   it('phone number should be E.164', async () => {
@@ -667,7 +678,5 @@ describe('SendgridAudiences.syncAudience', () => {
     expect(toDateFormat(badDate2)).toBe(undefined)
     expect(toDateFormat(badDate3)).toBe(undefined)
   })
-
-
 
 })

packages/destination-actions/src/destinations/sendgrid-audiences/syncAudience/index.ts

@@ -28,10 +28,10 @@ const action: ActionDefinition<Settings, Payload> = {
     }
   },
   perform: async (request, { payload }) => {
-    return await send(request, [payload])
+    return await send(request, [payload], false)
   },
   performBatch: async (request, { payload }) => {
-    return await send(request, payload)
+    return await send(request, payload, true)
   }
 }