fix: dedupe batch writes (#2)

We have duplicate multihash+car entries. We have to dedupe these from batch write commands or dynamo rejects it. Conceptually, it is redundant to track multiple instances of the same block appearing in the same CAR so it is fine and good to dedupe here License: MIT Signed-off-by: Oli Evans <[email protected]>
web3-storage · Sep 1, 2023 · 6cd9c55 · 6cd9c55
1 parent 33bc598
commit 6cd9c55
Show file tree

Hide file tree

Showing 2 changed files with 10 additions and 1 deletion.
diff --git a/write-cli.js b/write-cli.js
@@ -1,3 +1,5 @@
+#!/usr/bin/env node
+
 import { DynamoDBClient } from '@aws-sdk/client-dynamodb'
 import { createDynamo, createDynamoTable } from './test/_helpers.js'
 import { write } from './write.js'

diff --git a/write.js b/write.js
@@ -36,14 +36,21 @@ export async function write (srcStream, dst, segment, totalSegments, client = ne
         srcCount += batch.length
         spinner.suffixText = `src: ${srcCount} dst: ${dstCount}`
 
+        // remove duplicates
+        const itemMap = new Map()
+        for (const item of batch) {
+          itemMap.set(`${item.blockmultihash}#${item.carpath}`, item)
+        }
+
         /** @type {Array<import('@aws-sdk/client-dynamodb').PutRequest} */
-        const puts = batch.map(item => {
+        const puts = Array.from(itemMap.values()).map(item => {
           return {
             PutRequest: {
               Item: marshall(item)
             }
           }
         })
+
         const cmd = new BatchWriteItemCommand({
           RequestItems: {
             [dst]: puts