Check for null items in the delete iterator

cognitedata · Oct 16, 2023 · bd7c59a · bd7c59a
1 parent 231dd64
commit bd7c59a
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/main/scala/cognite/spark/v1/AssetHierarchyBuilder.scala b/src/main/scala/cognite/spark/v1/AssetHierarchyBuilder.scala
@@ -80,7 +80,9 @@ class AssetHierarchyBuilder(config: RelationConfig)(val sqlContext: SQLContext)
 
   def delete(data: DataFrame): Unit =
     data.foreachPartition((rows: Iterator[Row]) => {
-      val deletes = rows.map(r => fromRow[DeleteItemByCogniteId](r))
+      val deletes = rows
+        .map(fromRow[DeleteItemByCogniteId](_))
+        .filter(_ != null)
       Stream
         .fromIterator[IO](deletes, chunkSize = batchSize)
         .chunks