Adding Acquire to benchmark

CMakeLists.txt

@@ -1,22 +1,29 @@
-cmake_minimum_required(VERSION 3.2)
+cmake_minimum_required(VERSION 3.15)
 project(cpp-zarr)
 cmake_policy(SET CMP0057 NEW)
 
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED True)
 
-# finding the cppZarr lib
+# Find libraries and packages
 find_library(cppZarrLib cppZarr REQUIRED)
+find_package(blosc CONFIG REQUIRED)
+find_package(miniocpp CONFIG REQUIRED)
+find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
+find_package(pybind11 CONFIG REQUIRED)
+
+# Set up acquire-zarr to link against
+set(azlib "${CMAKE_CURRENT_LIST_DIR}/zarr_libraries/acquire/lib/libacquire-zarr.so")
 
-# setting include dir for dependencies
+# Setting include directories for cpp-zarr
 include_directories(/usr/local/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/zarr_libraries/acquire/include)
 
-# pybind11
-find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
-include_directories(${Python3_INCLUDE_DIRS})
+# Define pybind11 modules
+pybind11_add_module(pyCppZarr zarr_libraries/cpp_zarr/cpp_zarr.cpp)
+target_link_libraries(pyCppZarr PRIVATE ${cppZarrLib})
 
-find_package(pybind11 REQUIRED)
-include_directories(${pybind11_INCLUDE_DIR})
+pybind11_add_module(pyAcquire zarr_libraries/acquire/acquire.cpp)
 
-pybind11_add_module(pyCppZarr zarr_libraries/cpp_zarr/cpp_zarr.cpp)
-target_link_libraries(pyCppZarr PRIVATE ${cppZarrLib})
+# target_include_directories(pyAcquire PRIVATE ${CMAKE_CURRENT_LIST_DIR}/zarr_libraries/acquire/include)
+target_link_libraries(pyAcquire PRIVATE miniocpp::miniocpp blosc_static ${azlib})

CMakePresets.json

@@ -0,0 +1,12 @@
+{
+  "version": 3,
+  "configurePresets": [
+    {
+      "name": "default",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "CMAKE_TOOLCHAIN_FILE": "$env{VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
+      }
+    }
+  ]
+}

Dockerfile

@@ -22,6 +22,11 @@ RUN git clone https://github.com/abcucberkeley/cpp-zarr && \
     make -j && \
     make install
 
+WORKDIR /app
+
+RUN git clone https://github.com/microsoft/vcpkg.git && \
+    cd vcpkg && ./bootstrap-vcpkg.sh
+
 RUN pip install "pybind11[global]"
 
 # set directory for code base
@@ -39,15 +44,16 @@ RUN echo "source activate benchmark" > ~/.bashrc
 # set conda to path for working in docker terminal
 ENV PATH=/opt/conda/envs/benchmark/bin:$PATH
 
+# set vcpkg to path for cmake
+ENV VCPKG_ROOT=/path/to/vcpkg
+ENV PATH=$VCPKG_ROOT:$PATH
+
 # copy source code into docker app
 COPY . .
 
 # build c++ code 
-RUN mkdir build && \
-    cd build && \
-    cmake .. && \
-    make && \
-    cd ..
+RUN cmake --preset=default && \
+    cmake --build build
 
 # running the benchmark
 CMD [ "python", "main.py" ]

main.py

@@ -7,7 +7,7 @@ def main() -> None:
     benchmark = Benchmark(shape=[64, 1080, 1920], chunks=[64, 540, 960])
 
     benchmark.run_all_tests(
-        append_test_gigabytes=50, write_test_gigabytes=5,
+        append_test_gigabytes=50, write_test_gigabytes=5, choose_lib="Cpp Zarr",
         append_graph=graph[1][0], append_avg_graph=graph[1][1],
         write_graph=graph[0][0], write_avg_graph=graph[0][1]
     )

setup.sh

@@ -10,15 +10,6 @@ else
   echo "Directory '$BUILD_DIR' does not exist. No need to delete."
 fi
 
-# create a new directory
-echo "Creating a new '$BUILD_DIR' directory..."
-mkdir "$BUILD_DIR"
-
-echo "Directory '$BUILD_DIR' has been created successfully."
-
 # build cpp code 
-cd "$BUILD_DIR" || { echo "Failed to change directory to '$BUILD_DIR'"; exit 1; }
-cmake .. || { echo "Failed to run cmake '$BUILD_DIR'"; exit 1; }
-make || { echo "Failed to run make"; exit 1; }
-
-echo "Build process completed successfully."
+cmake --preset=default || { echo "Failed to run cmake '$BUILD_DIR'"; exit 1; }
+cmake --build build || { echo "Failed to build"; exit 1; }

vcpkg-configuration.json

@@ -0,0 +1,14 @@
+{
+  "default-registry": {
+    "kind": "git",
+    "baseline": "f7423ee180c4b7f40d43402c2feb3859161ef625",
+    "repository": "https://github.com/microsoft/vcpkg"
+  },
+  "registries": [
+    {
+      "kind": "artifact",
+      "location": "https://github.com/microsoft/vcpkg-ce-catalog/archive/refs/heads/main.zip",
+      "name": "microsoft"
+    }
+  ]
+}

vcpkg.json

@@ -0,0 +1,16 @@
+{
+  "dependencies": [
+    {
+      "name": "blosc",
+      "version>=": "1.21.5"
+    },
+    {
+      "name": "nlohmann-json",
+      "version>=": "3.11.3"
+    },
+    {
+      "name": "minio-cpp",
+      "version>=": "0.3.0"
+    }
+  ]
+}

zarr_libraries/acquire/acquire.cpp

@@ -0,0 +1,94 @@
+#include "zarr.h"
+
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <string>
+#include <time.h>
+#include <chrono>
+#include <cstdio>
+#include <vector>
+
+#define SIZED(str) str, sizeof(str)
+
+namespace py = pybind11;
+using namespace std::chrono;
+using namespace std;
+
+float write_zarr(string path, vector<uint64_t> chunks, vector<uint64_t> shape)
+{
+    Zarr_set_log_level(LogLevel_Debug);
+
+    ZarrStreamSettings *settings = ZarrStreamSettings_create();
+    ZarrStreamSettings_set_store_path(settings, path.c_str(), sizeof(path.c_str()));
+    ZarrStreamSettings_set_data_type(settings, ZarrDataType_uint8);
+
+    ZarrStreamSettings_set_compressor(settings, ZarrCompressor_Blosc1);
+    ZarrStreamSettings_set_compression_codec(settings,
+                                             ZarrCompressionCodec_BloscLZ4);
+    ZarrStreamSettings_set_compression_level(settings, 1);
+    ZarrStreamSettings_set_compression_shuffle(settings, 1);
+
+    ZarrStreamSettings_reserve_dimensions(settings, 3);
+    ZarrStreamSettings_set_dimension(settings,
+                                     0,
+                                     SIZED("t"),
+                                     ZarrDimensionType_Time,
+                                     shape[0],
+                                     chunks[0], // chunk size; could be whatever you want here
+                                     0);
+    ZarrStreamSettings_set_dimension(settings,
+                                     1,
+                                     SIZED("y"),
+                                     ZarrDimensionType_Space,
+                                     shape[1],
+                                     chunks[1], // chunk size; could be whatever you want here
+                                     0);
+    ZarrStreamSettings_set_dimension(settings,
+                                     2,
+                                     SIZED("x"),
+                                     ZarrDimensionType_Space,
+                                     shape[2],
+                                     chunks[2], // chunk size; could be whatever you want here
+                                     0);
+
+    ZarrStream *stream = ZarrStream_create(settings, ZarrVersion_2);
+
+    // acquire
+    const size_t chunk_size = shape[0] * shape[1] * shape[2];
+    std::vector<uint8_t> chunk(chunk_size); // one chunk's worth of data, fill this however you want
+
+    // filling with random data
+    for (uint8_t &val : chunk)
+    {
+        val = (uint8_t)(rand() % (UINT8_MAX + 1));
+    }
+
+    // write out a single chunk. Wrap this in a for loop to write out as many chunks as you want
+    size_t bytes_written;
+
+    auto begin_time = high_resolution_clock::now();
+    ZarrError status = ZarrStream_append(stream, chunk.data(), chunk_size * sizeof(uint8_t), &bytes_written);
+    duration<float, ratio<1>> duration = high_resolution_clock::now() - begin_time;
+
+    if (status != ZarrError_Success)
+    {
+        printf("Error: %s\n", Zarr_get_error_message(status));
+        return 1;
+    }
+
+    if (bytes_written != chunk_size * sizeof(uint8_t))
+    {
+        printf("Error: wrote %zu bytes, expected %zu\n", bytes_written, chunk_size * sizeof(uint8_t));
+        return 1;
+    }
+
+    // cleanup
+    ZarrStream_destroy(stream);
+
+    return duration.count();
+}
+
+PYBIND11_MODULE(pyAcquire, handle)
+{
+    handle.def("write_zarr", &write_zarr);
+}

zarr_libraries/acquire/acquire.py

@@ -0,0 +1,17 @@
+from build.pyAcquire import *
+from pathlib import Path
+
+
+class Acquire:
+    def __init__(self) -> None:
+        self.__path_to_data = str((Path(__file__).parent / "../example_data/acquire_data/test.zarr").resolve())
+
+
+    @property
+    def data_path(self) -> str:
+        return self.__path_to_data
+
+
+    def write_zarr(self, shape: list, chunks: list) -> float:
+        return write_zarr(self.data_path, chunks, shape)
+