adding ground work for cpp-zarr tests

.gitignore

@@ -1,4 +1,6 @@
 .DS_Store
-zarr_libraries/example_data/*
 __pycache__/
-.vscode/
+.vscode/
+example_data/
+build/
+test.*

CMakeLists.txt

@@ -0,0 +1,22 @@
+cmake_minimum_required(VERSION 3.2)
+project(cpp-zarr)
+cmake_policy(SET CMP0057 NEW)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED True)
+
+# finding the cppZarr lib
+find_library(cppZarrLib cppZarr REQUIRED)
+
+# setting include dir for dependencies
+include_directories(/usr/local/include)
+
+# pybind11
+find_package(Python3 REQUIRED COMPONENTS Interpreter Development)
+include_directories(${Python3_INCLUDE_DIRS})
+
+find_package(pybind11 REQUIRED)
+include_directories(${pybind11_INCLUDE_DIR})
+
+pybind11_add_module(pyCppZarr zarr_libraries/cpp_zarr/cpp_zarr.cpp)
+target_link_libraries(pyCppZarr PRIVATE ${cppZarrLib})

Dockerfile

@@ -3,8 +3,9 @@ FROM continuumio/miniconda3
 # set directory to root for insatlling dependencies
 WORKDIR /app
 
-# install system dependencies including OpenGL
+# install system dependencies 
 ENV DEBIAN_FRONTEND=noninteractive
+
 RUN apt-get update && apt-get install -y \
     python3-opengl \
     gcc \
@@ -21,6 +22,8 @@ RUN git clone https://github.com/abcucberkeley/cpp-zarr && \
     make -j && \
     make install
 
+RUN pip install "pybind11[global]"
+
 # set directory for code base
 WORKDIR /app/benchmark
 

environment.yml

@@ -6,11 +6,11 @@ channels:
 dependencies:
   - pip
   - matplotlib
-  - napari
   - pyqt
   - ome-zarr
   - kvikio
   - zarr=2.15*
   - pip:
+      - cmake
       - ml-dtypes
       - tensorstore

main.py

@@ -20,11 +20,11 @@ def run_all_tests(shape: list, chunks: list) -> None:
             * Zarr Python
         '''
         bandwidth_map["TensorStore Append"] = (
-            tensorstore.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=200)
-        ) 
-        bandwidth_map["Zarr Python Append"] = (
-            zarr_python.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=200)
+            tensorstore.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=100)
         ) 
+        #bandwidth_map["Zarr Python Append"] = (
+        #    zarr_python.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=100)
+        #) 
 
         # setting up graph for append tests
         graph[1][0].set_xlabel("Write Number")
@@ -38,7 +38,7 @@ def run_all_tests(shape: list, chunks: list) -> None:
             * TensorStore
             * Zarr Python
             * OME Zarr
-        '''
+        
         bandwidth_map["TensorStore Write"] = (
             tensorstore.continuous_write_test(graph=graph[0][0], avg_graph=graph[0][1], append_dim_size=51, step=5)
         )   
@@ -49,7 +49,7 @@ def run_all_tests(shape: list, chunks: list) -> None:
         bandwidth_map["OME Zarr Write"] = (
             ome_zarr.continuous_write_test(graph=graph[0][0], avg_graph=graph[0][1], append_dim_size=46, step=5)
         )      
-
+        '''
         # print the average bandwidth for each of the tests
         print(f"Shape {shape}, Chunks {chunks}")
         print("----------Bandwidth----------")

zarr_libraries/__init__.py

@@ -1,4 +1,7 @@
 from zarr_libraries.common import * 
 from zarr_libraries.tensorstore.tensorstore_zarr import *
 from zarr_libraries.zarr_python.zarr_python import *
-from zarr_libraries.ome_ngff.ome_zarr import *
+from zarr_libraries.ome_ngff.ome_zarr import *
+# pybind11
+import build.pyCppZarr
+from build.pyCppZarr import *

zarr_libraries/common.py

@@ -1,14 +1,5 @@
-import napari
-import dask.array as da
 import os
 
-
-# viewing data with napari
-def view_zarr(folder_path: str) -> None:
-    data = da.from_zarr(folder_path)
-    napari.view_image(data, colormap='magma')
-    napari.run()
-
 
 # getting size of zarr folder recursively 
 def folder_size(folder_path: str) -> int:

zarr_libraries/cpp_zarr/cpp_zarr.cpp

@@ -0,0 +1,61 @@
+#include <iostream>
+#include <vector>
+#include <filesystem>
+#include <time.h>
+#include "zarr.h"
+#include "parallelwritezarr.h"
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+using namespace std;
+
+class Cpp_Zarr
+{
+public:
+    Cpp_Zarr() = default;
+
+    int write_zarr()
+    {
+        const vector<uint64_t> startCoords{0, 0, 0};
+        const vector<uint64_t> endCoords{64, 1920, 1920};
+        const vector<uint64_t> writeShape({endCoords[0] - startCoords[0],
+                                           endCoords[1] - startCoords[1],
+                                           endCoords[2] - startCoords[2]});
+
+        // random data
+        size_t arrSize = writeShape[0] * writeShape[1] * writeShape[2];
+        void *zarrArr = malloc(arrSize * sizeof(uint8_t));
+        srand((unsigned int)time(NULL));
+        for (size_t i = 0; i < arrSize; i++)
+        {
+            ((uint8_t *)zarrArr)[i] = (uint8_t)(rand() % (UINT8_MAX + 1));
+        }
+
+        // creat zarr object
+        zarr zarrObject;
+
+        zarrObject.set_fileName("/home/chris/code/zarr-writers-benchmark/test.zarr");
+        zarrObject.set_dtype("<u1");
+        zarrObject.set_shape(writeShape);
+        zarrObject.set_chunks({64, 256, 256});
+        zarrObject.set_chunkInfo(startCoords, endCoords);
+        zarrObject.set_fill_value(1);
+        zarrObject.set_order("C");
+        zarrObject.set_dimension_separator("/");
+        zarrObject.write_zarray();
+
+        parallelWriteZarr(zarrObject, zarrArr, startCoords, endCoords, writeShape, 8, true, false);
+
+        free(zarrArr);
+
+        return 0;
+    }
+};
+
+PYBIND11_MODULE(pyCppZarr, handle)
+{
+    py::class_<Cpp_Zarr>(
+        handle, "Cpp_Zarr")
+        .def(py::init<>())
+        .def("write_zarr", &Cpp_Zarr::write_zarr);
+}