Don't materialize batches seq

cognitedata · Dec 9, 2024 · 3b93822 · 3b93822
1 parent 856c173
commit 3b93822
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/src/main/scala/cognite/spark/v1/RawTableRelation.scala b/src/main/scala/cognite/spark/v1/RawTableRelation.scala
@@ -294,9 +294,7 @@ class RawTableRelation(
           // start on the next batch (this limitation used to be per partition). Instead, we should
           // have a cats.effect.std.Semaphore permit with X number of outstanding requests
           // or cats.effect.concurrent.Backpressure.
-          rows
-            .grouped(batchSize)
-            .toSeq
+          (rows.grouped(batchSize): Iterator[Seq[Row]])
             .grouped(maxOutstandingRawInsertRequests)
             .foreach { batch =>
               batch.toVector