diff --git a/.gitignore b/.gitignore index 6beb076..db72643 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ .DS_Store -zarr_libraries/example_data/* __pycache__/ -.vscode/ \ No newline at end of file +.vscode/ +example_data/ +build/ +test.* \ No newline at end of file diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d0dd2ef --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,22 @@ +cmake_minimum_required(VERSION 3.2) +project(cpp-zarr) +cmake_policy(SET CMP0057 NEW) + +set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED True) + +# finding the cppZarr lib +find_library(cppZarrLib cppZarr REQUIRED) + +# setting include dir for dependencies +include_directories(/usr/local/include) + +# pybind11 +find_package(Python3 REQUIRED COMPONENTS Interpreter Development) +include_directories(${Python3_INCLUDE_DIRS}) + +find_package(pybind11 REQUIRED) +include_directories(${pybind11_INCLUDE_DIR}) + +pybind11_add_module(pyCppZarr zarr_libraries/cpp_zarr/cpp_zarr.cpp) +target_link_libraries(pyCppZarr PRIVATE ${cppZarrLib}) \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6774b78..5156186 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,9 @@ FROM continuumio/miniconda3 # set directory to root for insatlling dependencies WORKDIR /app -# install system dependencies including OpenGL +# install system dependencies ENV DEBIAN_FRONTEND=noninteractive + RUN apt-get update && apt-get install -y \ python3-opengl \ gcc \ @@ -21,6 +22,8 @@ RUN git clone https://github.com/abcucberkeley/cpp-zarr && \ make -j && \ make install +RUN pip install "pybind11[global]" + # set directory for code base WORKDIR /app/benchmark diff --git a/environment.yml b/environment.yml index be5ec70..81d3a3f 100644 --- a/environment.yml +++ b/environment.yml @@ -6,11 +6,11 @@ channels: dependencies: - pip - matplotlib - - napari - pyqt - ome-zarr - kvikio - zarr=2.15* - pip: + - cmake - ml-dtypes - tensorstore diff --git a/main.py b/main.py index 9322cdc..ca1a02f 100644 --- a/main.py +++ b/main.py @@ -20,11 +20,11 @@ def run_all_tests(shape: list, chunks: list) -> None: * Zarr Python ''' bandwidth_map["TensorStore Append"] = ( - tensorstore.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=200) - ) - bandwidth_map["Zarr Python Append"] = ( - zarr_python.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=200) + tensorstore.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=100) ) + #bandwidth_map["Zarr Python Append"] = ( + # zarr_python.continuous_append_test(graph=graph[1][0], avg_graph=graph[1][1], append_dim_size=100) + #) # setting up graph for append tests graph[1][0].set_xlabel("Write Number") @@ -38,7 +38,7 @@ def run_all_tests(shape: list, chunks: list) -> None: * TensorStore * Zarr Python * OME Zarr - ''' + bandwidth_map["TensorStore Write"] = ( tensorstore.continuous_write_test(graph=graph[0][0], avg_graph=graph[0][1], append_dim_size=51, step=5) ) @@ -49,7 +49,7 @@ def run_all_tests(shape: list, chunks: list) -> None: bandwidth_map["OME Zarr Write"] = ( ome_zarr.continuous_write_test(graph=graph[0][0], avg_graph=graph[0][1], append_dim_size=46, step=5) ) - + ''' # print the average bandwidth for each of the tests print(f"Shape {shape}, Chunks {chunks}") print("----------Bandwidth----------") diff --git a/zarr_libraries/__init__.py b/zarr_libraries/__init__.py index b73f245..eca20ee 100644 --- a/zarr_libraries/__init__.py +++ b/zarr_libraries/__init__.py @@ -1,4 +1,7 @@ from zarr_libraries.common import * from zarr_libraries.tensorstore.tensorstore_zarr import * from zarr_libraries.zarr_python.zarr_python import * -from zarr_libraries.ome_ngff.ome_zarr import * \ No newline at end of file +from zarr_libraries.ome_ngff.ome_zarr import * +# pybind11 +import build.pyCppZarr +from build.pyCppZarr import * \ No newline at end of file diff --git a/zarr_libraries/common.py b/zarr_libraries/common.py index e182f56..09006c8 100644 --- a/zarr_libraries/common.py +++ b/zarr_libraries/common.py @@ -1,14 +1,5 @@ -import napari -import dask.array as da import os - -# viewing data with napari -def view_zarr(folder_path: str) -> None: - data = da.from_zarr(folder_path) - napari.view_image(data, colormap='magma') - napari.run() - # getting size of zarr folder recursively def folder_size(folder_path: str) -> int: diff --git a/zarr_libraries/cpp_zarr/cpp_zarr.cpp b/zarr_libraries/cpp_zarr/cpp_zarr.cpp new file mode 100644 index 0000000..27ab4de --- /dev/null +++ b/zarr_libraries/cpp_zarr/cpp_zarr.cpp @@ -0,0 +1,61 @@ +#include +#include +#include +#include +#include "zarr.h" +#include "parallelwritezarr.h" +#include + +namespace py = pybind11; +using namespace std; + +class Cpp_Zarr +{ +public: + Cpp_Zarr() = default; + + int write_zarr() + { + const vector startCoords{0, 0, 0}; + const vector endCoords{64, 1920, 1920}; + const vector writeShape({endCoords[0] - startCoords[0], + endCoords[1] - startCoords[1], + endCoords[2] - startCoords[2]}); + + // random data + size_t arrSize = writeShape[0] * writeShape[1] * writeShape[2]; + void *zarrArr = malloc(arrSize * sizeof(uint8_t)); + srand((unsigned int)time(NULL)); + for (size_t i = 0; i < arrSize; i++) + { + ((uint8_t *)zarrArr)[i] = (uint8_t)(rand() % (UINT8_MAX + 1)); + } + + // creat zarr object + zarr zarrObject; + + zarrObject.set_fileName("/home/chris/code/zarr-writers-benchmark/test.zarr"); + zarrObject.set_dtype("( + handle, "Cpp_Zarr") + .def(py::init<>()) + .def("write_zarr", &Cpp_Zarr::write_zarr); +}