diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..14b61b5 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,31 @@ +name: publish dotnet packages + +on: + release: + types: [ created ] + +jobs: + deploy: + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Checkout the repository + uses: actions/checkout@v4 + - name: Setup .NET Core 8.x + uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 6.x + 8.x + cache: true + - name: Download chdb library + run: ./update_libchdb.sh + - run: dotnet build --configuration Release + - name: Create the packages + run: dotnet pack --configuration Release --include-symbols + - name: Publish the package to nuget.org + run: dotnet nuget push nupkg/*.nupkg -k $NUGET_AUTH_TOKEN -s https://api.nuget.org/v3/index.json + env: + NUGET_AUTH_TOKEN: ${{secrets.NUGET_TOKEN}} diff --git a/.github/workflows/dotnet.yml b/.github/workflows/dotnet.yml new file mode 100644 index 0000000..ac32337 --- /dev/null +++ b/.github/workflows/dotnet.yml @@ -0,0 +1,169 @@ +# This workflow will build a .NET project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-net + +name: build + +env: + DOTNET_NOLOGO: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + workflow_dispatch: + inputs: + reason: + description: 'The reason for running the workflow' + required: true + default: 'Manual run' + +jobs: + build_chdb: + name: Build chdb-${{ matrix.rid }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: ubuntu-latest + rid: linux-x64 + - os: macos-latest + rid: osx-x64 + - os: macos-14 + rid: osx-arm64 + + steps: + - name: 'Print manual run reason' + if: ${{ github.event_name == 'workflow_dispatch' }} + run: | + echo 'Reason: ${{ github.event.inputs.reason }}' + + - uses: actions/checkout@v4 + + - name: Setup .NET Core + uses: actions/setup-dotnet@v3 + with: + dotnet-version: | + 6.0.x + 8.0.x + source-url: https://nuget.pkg.github.com/vilinski/index.json + env: + NUGET_AUTH_TOKEN: ${{secrets.GITHUB_TOKEN}} + - name: Display dotnet version + run: | + dotnet --version + echo "GITHUB_WORKSPACE $GITHUB_WORKSPACE" + echo "GITHUB_ACTION $GITHUB_ACTION" + echo "GITHUB_RUN_ID $GITHUB_RUN_ID" + echo "GITHUB_RUN_NUMBER $GITHUB_RUN_NUMBER" + + - name: Restore + run: dotnet restore + + - name: Download chdb library + run: ./update_libchdb.sh + + - name: Build + run: | + # copy to the correct location + cp libchdb.so src/chdb/libchdb.so + ls -lahS src/chdb/libchdb* + dotnet build --no-restore --configuration Release + + - name: Test + run: dotnet test -c Release --no-build --logger trx --results-directory "TestResults-${{ matrix.rid }}" + + # - name: Upload dotnet test results + # uses: actions/upload-artifact@v4 + # with: + # name: dotnet-results-${{ matrix.rid }} + # path: TestResults-${{ matrix.rid }} + # # Use always() to always run this step to publish test results when there are test failures + # if: ${{ always() }} + + # - name: Pack chdb-${{ matrix.rid }} + # run: | + # dotnet pack src/chdb/chdb.csproj -c Release + # ls -lahS nupkg + + # - name: Publish the package to nuget.org + # run: dotnet nuget push nupkg/*.nupkg --skip-duplicate --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_AUTH_TOKEN_CHDB }} + # env: + # NUGET_AUTH_TOKEN: ${{ secrets.NUGET_AUTH_TOKEN_CHDB }} + + # - name: Publish chdb-${{ matrix.rid }} package to GPR + # run: dotnet nuget push nupkg/chdb-${{ matrix.rid }}.*.nupkg --skip-duplicate --api-key ${{ secrets.PACKAGES_TOKEN }} --source https://nuget.pkg.github.com/chdb-io/index.json + # env: + # NUGET_AUTH_TOKEN: ${{ secrets.PACKAGES_TOKEN }} + + # - name: Upload nupkg + # #run: ls -l nupkg/*.nupkg + # uses: actions/upload-artifact@v4 + # with: + # name: dotnet-nupkg-${{ matrix.rid }} + # path: nupkg + + push_chdb: + if: github.event.pull_request.merged + name: Push chdb + needs: build_chdb + runs-on: ubuntu-latest + steps: + + - uses: actions/checkout@v4 + + - name: Download chdb library + run: ./update_libchdb.sh + + # - name: Upload libchdb Artifact + # uses: actions/upload-artifact@v4 + # with: + # name: libchdb + # path: libchdb.so + + - name: Pack + run: | + cp libchdb.so src/chdb/libchdb.so + ls -lahS src/chdb/libchdb* + dotnet pack src/chdb/chdb.csproj -c Release --include-symbols + ls -lahS nupkg + + - name: Publish the package to nuget.org + run: dotnet nuget push nupkg/chdb.*.nupkg --skip-duplicate --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_AUTH_TOKEN_CHDB }} + + + push_tool: + if: github.event.pull_request.merged + name: Push chdb-tool + needs: push_chdb + runs-on: ubuntu-latest + env: + PUSH_TOOL: true + steps: + - uses: actions/checkout@v4 + + - name: Download chdb library + run: ./update_libchdb.sh + + - name: Pack + run: | + ls -lahS . + ls -lahS src/chdb/* + cp libchdb.so src/chdb/ + ls -lahS src/chdb/libchdb* + dotnet nuget sources add -n chdb ./nupkg + dotnet pack src/chdb-tool/chdb-tool.csproj -c Release --include-symbols + ls -lahS nupkg + + - name: Publish + run: dotnet nuget push nupkg/chdb-tool.*.nupkg --skip-duplicate --source https://api.nuget.org/v3/index.json --api-key ${{ secrets.NUGET_AUTH_TOKEN_CHDB }} + + - name: Test chdb-tool + run: | + ./update_libchdb.sh + dotnet tool install --add-source ./nupkg --global chdb-tool + which chdb + cp libchdb.so /home/runner/.dotnet/tools/ + chdb --help + chdb "select version()" PrettyCompact diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..55d8c6b --- /dev/null +++ b/.gitignore @@ -0,0 +1,488 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. +## +## Get latest from `dotnet new gitignore` + +libchdb.tar.gz +chdb.h +src/chdb/libchdb.so + +# dotenv files +.env + +# User-specific files +*.rsuser +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Mono auto generated files +mono_crash.* + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +[Ww][Ii][Nn]32/ +[Aa][Rr][Mm]/ +[Aa][Rr][Mm]64/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ +[Ll]ogs/ + +# Visual Studio 2015/2017 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# Visual Studio 2017 auto generated files +Generated\ Files/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUnit +*.VisualState.xml +TestResult.xml +nunit-*.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# Benchmark Results +BenchmarkDotNet.Artifacts/ + +# .NET +project.lock.json +project.fragment.lock.json +artifacts/ + +# Tye +.tye/ + +# ASP.NET Scaffolding +ScaffoldingReadMe.txt + +# StyleCop +StyleCopReport.xml + +# Files built by Visual Studio +*_i.c +*_p.c +*_h.h +*.ilk +*.meta +*.obj +*.iobj +*.pch +*.pdb +*.ipdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*_wpftmp.csproj +*.log +*.tlog +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# Visual Studio Trace Files +*.e2e + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# AxoCover is a Code Coverage Tool +.axoCover/* +!.axoCover/settings.json + +# Coverlet is a free, cross platform Code Coverage Tool +coverage*.json +coverage*.xml +coverage*.info + +# Visual Studio code coverage results +*.coverage +*.coveragexml + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# Note: Comment the next line if you want to checkin your web deploy settings, +# but database connection strings (with potential passwords) will be unencrypted +*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# NuGet Symbol Packages +*.snupkg +# The packages folder can be ignored because of Package Restore +**/[Pp]ackages/* +# except build/, which is used as an MSBuild target. +!**/[Pp]ackages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/[Pp]ackages/repositories.config +# NuGet v3's project.json files produces more ignorable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt +*.appx +*.appxbundle +*.appxupload + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!?*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +orleans.codegen.cs + +# Including strong name files can present a security risk +# (https://github.com/github/gitignore/pull/2483#issue-259490424) +#*.snk + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm +ServiceFabricBackup/ +*.rptproj.bak + +# SQL Server files +*.mdf +*.ldf +*.ndf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings +*.rptproj.rsuser +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat +node_modules/ + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) +*.vbw + +# Visual Studio 6 auto-generated project file (contains which files were open etc.) +*.vbp + +# Visual Studio 6 workspace and project file (working project files containing files to include in project) +*.dsw +*.dsp + +# Visual Studio 6 technical files +*.ncb +*.aps + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# CodeRush personal settings +.cr/personal + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc + +# Cake - Uncomment if you are using it +# tools/** +# !tools/packages.config + +# Tabs Studio +*.tss + +# Telerik's JustMock configuration file +*.jmconfig + +# BizTalk build output +*.btp.cs +*.btm.cs +*.odx.cs +*.xsd.cs + +# OpenCover UI analysis results +OpenCover/ + +# Azure Stream Analytics local run output +ASALocalRun/ + +# MSBuild Binary and Structured Log +*.binlog + +# NVidia Nsight GPU debugger configuration file +*.nvuser + +# MFractors (Xamarin productivity tool) working folder +.mfractor/ + +# Local History for Visual Studio +.localhistory/ + +# Visual Studio History (VSHistory) files +.vshistory/ + +# BeatPulse healthcheck temp database +healthchecksdb + +# Backup folder for Package Reference Convert tool in Visual Studio 2017 +MigrationBackup/ + +# Ionide (cross platform F# VS Code tools) working folder +.ionide/ + +# Fody - auto-generated XML schema +FodyWeavers.xsd + +# VS Code files for those working on multiple tools +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +# Windows Installer files from build outputs +*.cab +*.msi +*.msix +*.msm +*.msp + +# JetBrains Rider +*.sln.iml +.idea + +## +## Visual studio for Mac +## + + +# globs +Makefile.in +*.userprefs +*.usertasks +config.make +config.status +aclocal.m4 +install-sh +autom4te.cache/ +*.tar.gz +tarballs/ +test-results/ + +# Mac bundle stuff +*.dmg +*.app + +# content below from: https://github.com/github/gitignore/blob/master/Global/macOS.gitignore +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +# content below from: https://github.com/github/gitignore/blob/master/Global/Windows.gitignore +# Windows thumbnail cache files +Thumbs.db +ehthumbs.db +ehthumbs_vista.db + +# Dump file +*.stackdump + +# Folder config file +[Dd]esktop.ini + +# Recycle Bin used on file shares +$RECYCLE.BIN/ + +# Windows Installer files +*.cab +*.msi +*.msix +*.msm +*.msp + +# Windows shortcuts +*.lnk + +# Vim temporary swap files +*.swp diff --git a/Directory.Build.props b/Directory.Build.props new file mode 100644 index 0000000..d3145e5 --- /dev/null +++ b/Directory.Build.props @@ -0,0 +1,43 @@ + + + linux-x64;osx-x64 + enable + enable + latest + ChDb + $(NoWarn);CS1591 + + + + True + true + ../../nupkg + 0.0.3.0$(GITHUB_RUN_NUMBER) + true + $(Version).$(BuildNumber) + vilinski + Apache-2.0 + chdb.png + clickhouse;chdb + README.md + https://github.com/vilinski/chdb + https://github.com/chdb-io/chdb-dotnet + git + Create + + + + true + true + true + true + true + true + true + + + + + + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..42c6e14 --- /dev/null +++ b/README.md @@ -0,0 +1,117 @@ +# chdb + +A .NET Core binding for [chdb](https://doc.chdb.io) library. + +![GitHub License](https://img.shields.io/github/license/chdb-io/chdb-dotnet) +![example workflow](https://github.com/chdb-io/chdb-dotnet/actions/workflows/dotnet.yml/badge.svg) +![NuGet Version](https://img.shields.io/nuget/vpre/chdb) +![NuGet Downloads](https://img.shields.io/nuget/dt/chdb) + +### Usage + +Running on platforms: linux, osx, windows, and architectures: x64, arm64. + +>Note for windows users - there is no windows bild in sight, but you can still use it in WSL. + +Currently the librairy is too large to be packed into a nuget package, so you need to install it manually. Use the [update_libchdb.sh](update_libchdb.sh) script to download the library for your platform and architecture. + +```bash +# download the latest version of the library - it takes a version as an optional argument +./update_libchdb.sh +# install the package to your project +dotnet add package chdb +``` + +Also place the library in appropriate folder, and add following to your csproj file: + +```xml + + + PreserveNewest + + +``` + +Then you can use it in your code like this: + +```csharp +using ChDb; + +var result = ChDb.Query("select version()"); +Console.WriteLine(result.Text); +// 23.10.1.1 +var s = new Session(); +var result = s.Query("select * from system.formats where is_output = 1", "PrettyCompact"); +// ┌─name───────────────────────────────────────┬─is_input─┬─is_output─┬─supports_parallel_parsing─┬─supports_parallel_formatting─┐ +// │ Prometheus │ 0 │ 1 │ 0 │ 0 │ +// │ PostgreSQLWire │ 0 │ 1 │ 0 │ 0 │ +// │ MySQLWire │ 0 │ 1 │ 0 │ 0 │ +// │ JSONEachRowWithProgress │ 0 │ 1 │ 0 │ 0 │ +// │ ODBCDriver2 │ 0 │ 1 │ 0 │ 0 │ +// ... +var result = s.Query("DESCRIBE s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/house_parquet/house_0.parquet')"); +Console.WriteLine(result.Text); +``` + +or use it right in F# interactive with `dotnet fsi`: + +```fsharp +#r "nuget: chdb" + +open ChDb + +// print out result in the PrettyCompact format by default +let result = ChDb.Query "select version()" +printfn "%s" result.Text +// or save result to a text or binary file in any supported format +let result = ChDb.Query("select * from system.formats where is_output = 1", "CSVWithNames") +System.IO.File.WriteAllBytes("supported_formats.csv", result.Buf) +``` + +## chdb-tool + +![NuGet Version](https://img.shields.io/nuget/vpre/chdb-tool) +![NuGet Downloads](https://img.shields.io/nuget/dt/chdb-tool) + +This is a dotnet tool, running [chdb](https://doc.chdb.io) library. +Probably you better served using the clickhouse client and run `clickhouse local`, but maybe it is more useful in some cases. + +### Installation + +Requires .NET SDK 6.0 or later. + +```bash +dotnet tool install --global chdb-tool +``` + +OS supported: linux, osx +ARCH supported: x64, arm64 + +### Usage + +Try any of this commands lines to see which output you get. + +```bash +chdb +chdb --version +chdb --help +chdb "select version()" +chdb "select * from system.formats where is_output = 1" PrettyCompact +``` + +# Build + +```bash +./update_libchdb.sh [v1.2.1] +cp libchdb.so src/chdb/ +dotnet build -c Release +dotnet test -c Release +dotnet pack -c Release +dotnet nuget add source ./nupkg --name chdb +dotnet tool update -g chdb-tool +chdb --version +``` + +## Authors + +* [Andreas Vilinski](https://github.com/vilinski) diff --git a/chdb.png b/chdb.png new file mode 100644 index 0000000..7b0136b Binary files /dev/null and b/chdb.png differ diff --git a/chdb.sln b/chdb.sln new file mode 100644 index 0000000..e5e647e --- /dev/null +++ b/chdb.sln @@ -0,0 +1,43 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.0.31903.59 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{53AE4A72-327B-44A3-BFAF-DF227D4D848A}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "chdb", "src\chdb\chdb.csproj", "{3B3A43FC-6DD0-4285-857C-271D19099613}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "chdb-tool", "src\chdb-tool\chdb-tool.csproj", "{78D934D4-0467-410A-83A7-AC9FF23C1E79}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{F34DFA50-9A9B-406A-8535-D705F29730F3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "test", "test\ChDbTest\test.csproj", "{458995B1-8DB5-40AC-9703-1049CA8A534B}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {3B3A43FC-6DD0-4285-857C-271D19099613}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {3B3A43FC-6DD0-4285-857C-271D19099613}.Debug|Any CPU.Build.0 = Debug|Any CPU + {3B3A43FC-6DD0-4285-857C-271D19099613}.Release|Any CPU.ActiveCfg = Release|Any CPU + {3B3A43FC-6DD0-4285-857C-271D19099613}.Release|Any CPU.Build.0 = Release|Any CPU + {78D934D4-0467-410A-83A7-AC9FF23C1E79}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {78D934D4-0467-410A-83A7-AC9FF23C1E79}.Debug|Any CPU.Build.0 = Debug|Any CPU + {78D934D4-0467-410A-83A7-AC9FF23C1E79}.Release|Any CPU.ActiveCfg = Release|Any CPU + {78D934D4-0467-410A-83A7-AC9FF23C1E79}.Release|Any CPU.Build.0 = Release|Any CPU + {458995B1-8DB5-40AC-9703-1049CA8A534B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {458995B1-8DB5-40AC-9703-1049CA8A534B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {458995B1-8DB5-40AC-9703-1049CA8A534B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {458995B1-8DB5-40AC-9703-1049CA8A534B}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {3B3A43FC-6DD0-4285-857C-271D19099613} = {53AE4A72-327B-44A3-BFAF-DF227D4D848A} + {78D934D4-0467-410A-83A7-AC9FF23C1E79} = {53AE4A72-327B-44A3-BFAF-DF227D4D848A} + {458995B1-8DB5-40AC-9703-1049CA8A534B} = {F34DFA50-9A9B-406A-8535-D705F29730F3} + EndGlobalSection +EndGlobal diff --git a/src/chdb-meta/chdb-meta.csproj b/src/chdb-meta/chdb-meta.csproj new file mode 100644 index 0000000..2b229b2 --- /dev/null +++ b/src/chdb-meta/chdb-meta.csproj @@ -0,0 +1,16 @@ + + + + chdb + chdb + chdb native bindings for dotnet core + + + + + + + + + + diff --git a/src/chdb-tool/Program.cs b/src/chdb-tool/Program.cs new file mode 100644 index 0000000..8a5483f --- /dev/null +++ b/src/chdb-tool/Program.cs @@ -0,0 +1,48 @@ +// See https://aka.ms/new-console-template for more information + +using System.Reflection; + +void PrintVersion() +{ + var versionString = Assembly.GetEntryAssembly()? + .GetCustomAttribute()? + .InformationalVersion ?? "unknown"; + Console.WriteLine($"chdb-tool v{versionString}"); +} + +void PrintHelp() +{ + PrintVersion(); + Console.WriteLine("-------------"); + Console.WriteLine("\nUsage:"); + Console.WriteLine(" chdb-tool "); +} + +if (args.Length == 0 || args[0] == "--help" || args[0] == "-h") +{ + PrintHelp(); +} +else if (args[0] == "--version" || args[0] == "-v") +{ + PrintVersion(); +} +else + try + { + var result = ChDb.ChDb.Query(args[0], args.Length > 1 && !args[1].StartsWith('-') ? args[1] : "PrettyCompact"); + if (result == null) + return; // TODO behavior changed in 1.2.1 + Console.WriteLine(result.Text); + if (!args.Contains("--quiet") && !args.Contains("-q")) + { + Console.WriteLine($"Elapsed: {result.Elapsed} s, read {result.RowsRead} rows, {result.BytesRead} bytes" + + $", {result.RowsRead / result.Elapsed.TotalSeconds:F0} rows/s, {result.BytesRead / result.Elapsed.TotalSeconds:F0} bytes/s"); + if (!string.IsNullOrWhiteSpace(result.ErrorMessage)) + Console.Error.WriteLine("Error message: " + result.ErrorMessage); + } + } + catch (ArgumentException e) + { + Console.Error.WriteLine(e.Message); + Environment.Exit(1); + } diff --git a/src/chdb-tool/chdb-tool.csproj b/src/chdb-tool/chdb-tool.csproj new file mode 100644 index 0000000..21322f5 --- /dev/null +++ b/src/chdb-tool/chdb-tool.csproj @@ -0,0 +1,26 @@ + + + + + + + + + + + net6.0;net8.0 + Exe + true + chdb + chdb-tool + + chdb tool + clickhouse local cli as a dotnet tool + + diff --git a/src/chdb/ChDb.cs b/src/chdb/ChDb.cs new file mode 100644 index 0000000..f3c2130 --- /dev/null +++ b/src/chdb/ChDb.cs @@ -0,0 +1,82 @@ +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security; + +namespace ChDb; + +internal static class NativeMethods +{ + private const string __DllName = "libchdb.so"; + + [DllImport(__DllName, EntryPoint = "query_stable_v2", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern IntPtr query_stable_v2(int argc, string[] argv); + + [DllImport(__DllName, EntryPoint = "free_result_v2", CallingConvention = CallingConvention.Cdecl, ExactSpelling = true)] + internal static extern void free_result_v2(IntPtr result); +} + +/// +/// Entry point for stateless chdb queries. +/// +public static class ChDb +{ + /// + /// Execute a stateless query and return the result. + /// + /// SQL query + /// Optional output format from supported by clickhouse. Default is TabSeparated. + /// Query result + /// + /// Stateless queries are useful for simple queries that do not require a session. + /// Use if you need to create databases or tables. + /// Set to a non-temporary directory to keep the data between sessions. + /// + public static LocalResult? Query(string query, string? format = null) + { + if (query is null) + throw new ArgumentNullException(nameof(query)); + var argv = new[] { + "clickhouse", + "--multiquery", + $"--query={query}", + $"--output-format={format ?? "TabSeparated"}" + }; + return Execute(argv); + } + + private static string? LookUpArg(string[] argv, string key) + { + foreach (var arg in argv) + { + var prefix = $"--{key}="; + if (arg.StartsWith(prefix)) + return arg.Substring(prefix.Length); + } + return null; + } + + [SuppressUnmanagedCodeSecurity] + [SecurityCritical] + internal static LocalResult? Execute(string[] argv) + { + try + { + var ptr = NativeMethods.query_stable_v2(argv.Length, argv); + var res = LocalResult.FromPtr(ptr); + NativeMethods.free_result_v2(ptr); + // Marshal.FreeHGlobal(ptr); + return res; + } + catch (RuntimeWrappedException e) + { + if (e.WrappedException is string s) + throw new ArgumentException($"Unmanaged string error {s}"); + else + throw new ArgumentException($"Unmanaged unknown error {e.WrappedException}"); + } + catch (Exception e) + { + throw new Exception("Managed error", e); + } + } +} \ No newline at end of file diff --git a/src/chdb/LocalResult.cs b/src/chdb/LocalResult.cs new file mode 100644 index 0000000..5beea70 --- /dev/null +++ b/src/chdb/LocalResult.cs @@ -0,0 +1,79 @@ +using System.Runtime.InteropServices; + +namespace ChDb; + +/// +/// The query result. +/// +public record LocalResult +{ + public byte[]? Buf { get; } + public string? ErrorMessage { get; } + /// + /// By text formats contains a result text. + /// + public string? Text => Buf == null ? null : System.Text.Encoding.UTF8.GetString(Buf); + public ulong RowsRead { get; } + public ulong BytesRead { get; } + public TimeSpan Elapsed { get; } + + /// Result buffer. + /// Error message if occured. + /// Number of rows read + /// Number of bytes read + /// Query time elapsed, in seconds. + public LocalResult(byte[]? Buf, string? ErrorMessage, ulong RowsRead, ulong BytesRead, TimeSpan Elapsed) + { + this.Buf = Buf; + this.ErrorMessage = ErrorMessage; + this.RowsRead = RowsRead; + this.BytesRead = BytesRead; + this.Elapsed = Elapsed; + } + + internal static LocalResult? FromPtr(nint ptr) + { + if (ptr == IntPtr.Zero) + return null; + var h = Marshal.PtrToStructure(ptr); + if (h == null) + return null; + + var errorMessage = h.error_message == IntPtr.Zero ? null : MarshalPtrToStringUTF8(h.error_message); + if (errorMessage != null) + return new LocalResult(null, errorMessage, 0, 0, TimeSpan.Zero); + + var elapsed = TimeSpan.FromSeconds(h.elapsed); + + var buf = h.buf == IntPtr.Zero ? null : new byte[h.len]; + if (buf != null) + Marshal.Copy(h.buf, buf, 0, h.len); + return new LocalResult(buf, errorMessage, h.rows_read, h.bytes_read, elapsed); + } + + private static string MarshalPtrToStringUTF8(nint ptr) + { + unsafe + { + var str = (byte*)ptr; + var length = 0; + for (var i = str; *i != 0; i++, length++) ; + var clrString = System.Text.Encoding.UTF8.GetString(str, length); + return clrString; + } + } + + [StructLayout(LayoutKind.Sequential)] + internal class Handle + { + internal nint buf; + internal int len; + internal nint _vec; // std::vector *, for freeing + internal double elapsed; + internal ulong rows_read; + internal ulong bytes_read; + internal nint error_message; + + public override string ToString() => $"Handle{{\n\tbuf={buf},\n\tlen={len},\n\t_vec={_vec},\n\telapsed={elapsed},\n\trows_read={rows_read},\n\tbytes_read={bytes_read},\n\terror_message={error_message}}}"; + } +} \ No newline at end of file diff --git a/src/chdb/Session.cs b/src/chdb/Session.cs new file mode 100644 index 0000000..fb3c082 --- /dev/null +++ b/src/chdb/Session.cs @@ -0,0 +1,52 @@ +namespace ChDb; + +public record Session : IDisposable +{ + /// + /// Output format for queries if not explicitely specified. Default is TabSeparated + /// + public string? Format { get; set; } + /// + /// Path to the ClickHouse data directory. If not set, a temporary directory will be used. + /// + public string? DataPath { get; set; } = Path.Combine(Path.GetTempPath(), "chdb_"); + /// + /// Query Log Level. + /// + public string? LogLevel { get; set; } + /// + /// Whether to delete the data directory on dispose. Default is true. + /// + public bool IsTemp { get; set; } = true; + + public void Dispose() + { + if (!IsTemp && DataPath?.EndsWith("chdb_") == true && Directory.Exists(DataPath)) + Directory.Delete(DataPath, true); + } + + /// + /// Execute a query and return the result. + /// + /// SQL query + /// Output format, optional. + /// Query result + public LocalResult? Query(string query, string? format = null) + { + if (IsTemp && DataPath is null) + { + DataPath = Path.Combine(Path.GetTempPath(), "chdb_"); + } + + var argv = new[] { + "clickhouse", + "--multiquery", + $"--query={query}", + $"--output-format={format ?? Format ?? "TabSeparated"}", //$"--path={DataPath}", + $"--path={DataPath}", + // $"--user_scripts_path={UdfPath}", $"--user_defined_executable_functions_config={UdfPath}/*.xml", + $"--log-level={LogLevel ?? "trace"}", + }; + return ChDb.Execute(argv); + } +} \ No newline at end of file diff --git a/src/chdb/chdb.csproj b/src/chdb/chdb.csproj new file mode 100644 index 0000000..58d8e8e --- /dev/null +++ b/src/chdb/chdb.csproj @@ -0,0 +1,19 @@ + + + + tests + netstandard2.1 + + chdb + chdb + chdb native bindings for dotnet core + true + + + + + + diff --git a/test/ChDbTest/ChDbTest.cs b/test/ChDbTest/ChDbTest.cs new file mode 100644 index 0000000..cec8f44 --- /dev/null +++ b/test/ChDbTest/ChDbTest.cs @@ -0,0 +1,204 @@ +namespace ChDb; + +[TestClass] +public class ChDbTest +{ + [TestMethod] + public void QueryVersionTest() + { + var result = ChDb.Query("select version()"); + Assert.IsNotNull(result); + Assert.AreEqual(1UL, result.RowsRead); + Assert.AreEqual(52UL, result.BytesRead); + Assert.AreEqual("23.10.1.1\n", result.Text); + Assert.IsNull(result.ErrorMessage); + Assert.AreNotEqual(TimeSpan.Zero, result.Elapsed); + } + + [TestMethod] + public void QueryErrorTest() + { + Assert.ThrowsException(() => ChDb.Query(null!)); + // TODO behavior changed in 1.2.1 + var r1 = ChDb.Query("wrong_query"); + Assert.IsNotNull(r1); + Assert.IsNull(r1.Text); + Assert.IsNotNull(r1.ErrorMessage); + + var r2 = ChDb.Query("wrong_query", "PrettyCompact"); + Assert.IsNotNull(r2); + Assert.IsNull(r2.Text); + Assert.IsNotNull(r2.ErrorMessage); + + var r3 = ChDb.Query("select version()", "wrong_format"); + Assert.IsNotNull(r3); + Assert.IsNull(r3.Text); + StringAssert.Contains(r3.ErrorMessage, "Unknown output format"); + } + + [TestMethod] + public void NoDataTest() + { + var result = ChDb.Query("create table x(a UInt8, b UInt8, c UInt8) Engine=Memory"); + Assert.IsNotNull(result); + Assert.AreEqual(0UL, result.RowsRead); + Assert.AreEqual(0UL, result.BytesRead); + Assert.IsNull(result.Text); + Assert.IsNull(result.ErrorMessage); + Assert.AreNotEqual(TimeSpan.Zero, result.Elapsed); + Assert.IsTrue(0.1 > result.Elapsed.TotalSeconds); + } + + [TestMethod] + public void EmptyResultTest() + { + var result = ChDb.Query("show tables"); + Assert.IsNotNull(result); + Assert.AreEqual(0UL, result.RowsRead); + Assert.AreEqual(0UL, result.BytesRead); + Assert.AreEqual("", result.Text); + Assert.IsNull(result.ErrorMessage); + Assert.AreNotEqual(TimeSpan.Zero, result.Elapsed); + Assert.IsTrue(0.1 > result.Elapsed.TotalSeconds); + } + + [TestMethod] + public void RowNumberTest() + { + var result = ChDb.Query("SELECT * FROM numbers(10)"); + Assert.IsNotNull(result); + Assert.AreEqual(10UL, result.RowsRead); + } + + [TestMethod] + public void FormatTest() + { + Assert.AreEqual("1\t2\t3\n", ChDb.Query("SELECT 1 as a, 2 b, 3 c")!.Text); + Assert.AreEqual("1,2,3\n", ChDb.Query("SELECT 1 as a, 2 b, 3 c", "CSV")!.Text); + Assert.AreEqual("\"a\",\"b\",\"c\"\n1,2,3\n", ChDb.Query("SELECT 1 as a, 2 b, 3 c", "CSVWithNames")!.Text); + StringAssert.Contains(ChDb.Query("SELECT 1 as a, 2 b, 3 c", "CSVWithNamesAndTypes")!.Text, "UInt8"); + } + + [TestMethod] + public void InMemoryTest() + { + var sql = + """ + create table test (a UInt8, b UInt8, c UInt8) Engine=Memory; + insert into test values (1, 2, 3); + select * from test; show tables; + drop table test;show tables + """; + var result = ChDb.Query(sql); + Assert.IsNotNull(result); + Assert.AreEqual("", result.Text); + Assert.AreEqual(null, result.ErrorMessage); + } + + [TestMethod] + public void S3ParquetTest() + { + var result = ChDb.Query("DESCRIBE s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/house_parquet/house_0.parquet')"); + Assert.IsNotNull(result); + Assert.IsNull(result.ErrorMessage); + StringAssert.StartsWith(result.Text, "price\tNullable(Int64)"); + } + + [TestMethod] + public void S3CountTest() + { + var result = ChDb.Query( + """ + SELECT count() + FROM s3('https://datasets-documentation.s3.eu-west-3.amazonaws.com/house_parquet/house_0.parquet') + """); + Assert.IsNotNull(result); + Assert.IsNull(result.ErrorMessage); + Assert.IsTrue(int.TryParse(result.Text, out var count)); + Assert.AreEqual(2772030, count); + } + + [TestMethod] + public void CsvTest() + { + var csv = """ + Name, Age, City + John, 25, New York + Alice, 30, London + Bob, 22, Tokyo + Eva, 28, Paris + """; + var dataPath = "/tmp/chdb/data"; + Directory.CreateDirectory(dataPath); + File.WriteAllText(Path.Combine(".", "test.csv"), csv); + var session = new Session + { + Format = "PrettyCompact", + DataPath = dataPath, + LogLevel = "trace", + }; + var result = session.Query("SELECT * FROM 'test.csv'", "CSVWithNamesAndTypes"); + Assert.IsNotNull(result); + Assert.AreEqual(4UL, result.RowsRead); + Assert.AreEqual(155UL, result.BytesRead); + StringAssert.StartsWith(result.Text, + """ + "Name","Age","City" + """); + } + + [TestMethod] + public void SessionTest() + { + using var s = new Session + { + Format = "PrettyCompact", + LogLevel = "trace", + }; + var nr = "xyz"; + + Assert.IsNull(s.Query($"select version()")?.ErrorMessage); + + // chdb creates "_local" database instead of "default" in clickhouse + StringAssert.Contains(s.Query($"SHOW DATABASES")?.Text, "_local"); + StringAssert.Contains(s.Query($"SELECT currentDatabase()")?.Text, "_local"); + Assert.AreEqual("", s.Query($"SHOW TABLES")?.Text); + + var r1 = s.Query($"DROP DATABASE IF EXISTS db_{nr}"); + Assert.IsNotNull(r1); + Assert.IsNull(r1.Text); + Assert.IsNull(r1.ErrorMessage); + + var r2 = s.Query($"CREATE DATABASE IF NOT EXISTS db_{nr} ENGINE = Atomic"); + Assert.IsNotNull(r2); + Assert.IsNull(r2.Text); + Assert.IsNull(r2.ErrorMessage); + + var r3 = s.Query($"CREATE TABLE IF NOT EXISTS db_{nr}.log_table_{nr} (x String, y Int) ENGINE = Log;"); + Assert.IsNotNull(r3); + Assert.IsNull(r3.Text); + Assert.IsNull(r3.ErrorMessage); + + var r4 = s.Query($"INSERT INTO db_{nr}.log_table_{nr} VALUES ('a', 1), ('b', 3), ('c', 2), ('d', 5);"); + Assert.IsNotNull(r4); + Assert.IsNull(r4.Text); + Assert.IsNull(r4.ErrorMessage); + + var r5 = s.Query($"SELECT * FROM db_{nr}.log_table_{nr}", "TabSeparatedWithNames"); + Assert.IsNotNull(r5); + Assert.AreEqual("x\ty\na\t1\nb\t3\nc\t2\nd\t5\n", r5.Text); + Assert.IsNull(r5.ErrorMessage); + + var r6 = s.Query($"CREATE VIEW db_{nr}.view_{nr} AS SELECT * FROM db_{nr}.log_table_{nr} LIMIT 4;"); + Assert.IsNotNull(r6); + Assert.IsNull(r6.Text); + Assert.IsNull(r6.ErrorMessage); + + var r7 = s.Query($"SELECT * FROM db_{nr}.view_{nr}", "TabSeparatedWithNames"); + Assert.IsNotNull(r7); + Assert.AreEqual("x\ty\na\t1\nb\t3\nc\t2\nd\t5\n", r7.Text); + Assert.IsNull(r7.ErrorMessage); + + s.Query($"DROP DATABASE IF EXISTS db_{nr}"); + } +} \ No newline at end of file diff --git a/test/ChDbTest/GlobalUsings.cs b/test/ChDbTest/GlobalUsings.cs new file mode 100644 index 0000000..ab67c7e --- /dev/null +++ b/test/ChDbTest/GlobalUsings.cs @@ -0,0 +1 @@ +global using Microsoft.VisualStudio.TestTools.UnitTesting; \ No newline at end of file diff --git a/test/ChDbTest/test.csproj b/test/ChDbTest/test.csproj new file mode 100644 index 0000000..3fe28b1 --- /dev/null +++ b/test/ChDbTest/test.csproj @@ -0,0 +1,20 @@ + + + + net6.0;net8.0 + false + true + + + + + + + + + + + + + + diff --git a/update_libchdb.sh b/update_libchdb.sh new file mode 100755 index 0000000..8d7a2ea --- /dev/null +++ b/update_libchdb.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +# Download the correct version based on the platform +case "$(uname -s)" in + Linux) + if [[ $(uname -m) == "aarch64" ]]; then + PLATFORM="linux-aarch64" + else + PLATFORM="linux-x86_64" + fi + ;; + Darwin) + if [[ $(uname -m) == "arm64" ]]; then + PLATFORM="macos-arm64" + else + PLATFORM="macos-x86_64" + fi + ;; + *) + echo "Unsupported platform" + exit 1 + ;; +esac + +# Get the newest release version +LATEST_RELEASE=$(curl --silent "https://api.github.com/repos/chdb-io/chdb/releases/latest" | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/') +RELEASE=${1:-$LATEST_RELEASE} + +DOWNLOAD_URL="https://github.com/chdb-io/chdb/releases/download/$RELEASE/$PLATFORM-libchdb.tar.gz" + +echo "Downloading $PLATFORM-libchdb.tar.gz from $DOWNLOAD_URL (latest is $LATEST_RELEASE)" + +# Download the file +curl -L -o libchdb.tar.gz "$DOWNLOAD_URL" + +# Untar the file +tar -xzf libchdb.tar.gz + +# Set execute permission for libchdb.so +chmod +x libchdb.so + +# Clean up +rm -f libchdb.tar.gz