Write Parquet file into a memory buffer (#612)

And drop `export_parquet()` wrapper
r-dbi · Jun 10, 2024 · ef6dc0e · ef6dc0e · hadley · Jun 10, 2024
1 parent 3642c14
commit ef6dc0e
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 17 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -29,7 +29,7 @@ Imports:
     prettyunits,
     rlang (>= 1.1.0),
     tibble,
-    nanoparquet
+    nanoparquet (> 0.2.0)
 Suggests:
     blob,
     covr,
@@ -41,6 +41,8 @@ Suggests:
     testthat (>= 3.1.5),
     wk (>= 0.3.2),
     withr
+Remotes:
+    r-lib/nanoparquet
 LinkingTo: 
     cli,
     cpp11,

diff --git a/R/bq-perform.R b/R/bq-perform.R
@@ -157,9 +157,10 @@ bq_perform_upload <- function(x, values,
       "content" = export_json(values)
     )
   } else {
+    # https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet?hl=es-419
     media <- list(
       "type" = "application/vnd.apache.parquet",
-      "content" = export_parquet(values)
+      "content" = nanoparquet::write_parquet(values, ":raw:")
     )
   }
 
@@ -199,21 +200,6 @@ export_json <- function(values) {
   rawToChar(rawConnectionValue(con))
 }
 
-# https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet?hl=es-419
-export_parquet <- function(values) {
-
-  tmpfile <- tempfile(fileext = ".parquet")
-
-  defer(unlink(tmpfile))
-
-  # write to disk
-  nanoparquet::write_parquet(values, tmpfile)
-
-  # read back results
-  readBin(tmpfile, what = "raw", n = file.info(tmpfile)$size)
-
-}
-
 #' @export
 #' @name api-perform
 #' @param source_uris The fully-qualified URIs that point to your data in