From 8408ebbbecb72bed8597dd717fed64b08126cb65 Mon Sep 17 00:00:00 2001 From: auxten Date: Mon, 17 Apr 2023 11:11:28 +0800 Subject: [PATCH 1/6] Fix version in __init__.py --- chdb/__init__.py | 4 ++-- setup.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/chdb/__init__.py b/chdb/__init__.py index a581f5eb3cb..9b8c83929e7 100644 --- a/chdb/__init__.py +++ b/chdb/__init__.py @@ -1,8 +1,7 @@ import sys import os -import pyarrow as pa -chdb_version = (0, 1, 0) +chdb_version = (0, 5, 0) if sys.version_info[:2] >= (3, 7): # get the path of the current file current_path = os.path.dirname(os.path.abspath(__file__)) @@ -26,6 +25,7 @@ def _to_arrowTable(res): """convert res to arrow table""" + import pyarrow as pa #import when needed return pa.RecordBatchFileReader(res.get_memview()).read_all() def to_df(r): diff --git a/setup.py b/setup.py index 355f4020798..c6403fd4ff9 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ import os import sys +import re import subprocess import sysconfig from setuptools import setup, Extension @@ -57,6 +58,20 @@ def get_latest_git_tag(minor_ver_auto=False): print(e) raise +# replace the version in chdb/__init__.py, which is `chdb_version = (0, 1, 0)` by default +# regex replace the version string `chdb_version = (0, 1, 0)` with version parts +def fix_version_init(version): + # split version string into parts + p1, p2, p3 = version.split('.') + init_file = os.path.join(script_dir, "chdb", "__init__.py") + with open(init_file, "r+") as f: + init_content = f.read() + # regex replace the version string `chdb_version = (0, 1, 0)` + regPattern = r"chdb_version = \(\d+, \d+, \d+\)" + init_content = re.sub(regPattern, f"chdb_version = ({p1}, {p2}, {p3})", init_content) + f.seek(0) + f.write(init_content) + # As of Python 3.6, CCompiler has a `has_flag` method. # cf http://bugs.python.org/issue26689 @@ -147,10 +162,12 @@ def build_extensions(self): extra_objects=[chdb_so], ), ] - + # fix the version in chdb/__init__.py + versionStr = get_latest_git_tag() + fix_version_init(versionStr) setup( packages=['chdb'], - version=get_latest_git_tag(), + version=versionStr, package_data={'chdb': [chdb_so]}, exclude_package_data={'': ['*.pyc', 'src/**']}, ext_modules=ext_modules, From 44aa79a8ec240b2be1c7e8c4d5edb0c2863d20ca Mon Sep 17 00:00:00 2001 From: auxten Date: Mon, 17 Apr 2023 16:15:07 +0800 Subject: [PATCH 2/6] Fix replace file content bug --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c6403fd4ff9..fd17a538c72 100644 --- a/setup.py +++ b/setup.py @@ -71,6 +71,7 @@ def fix_version_init(version): init_content = re.sub(regPattern, f"chdb_version = ({p1}, {p2}, {p3})", init_content) f.seek(0) f.write(init_content) + f.truncate() # As of Python 3.6, CCompiler has a `has_flag` method. From 15b071364afac442a9400e26d0d4edf4a7d9a2f9 Mon Sep 17 00:00:00 2001 From: auxten Date: Mon, 17 Apr 2023 18:35:34 +0800 Subject: [PATCH 3/6] Try import pyarrow and pandas when necessary --- .github/workflows/build_wheels.yml | 2 ++ chdb/__init__.py | 28 ++++++++++++++++++++-------- setup.py | 1 - 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 0c223ad343b..6acf8c38e08 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -82,6 +82,7 @@ jobs: export CC=/usr/bin/clang export CXX=/usr/bin/clang++ bash ./chdb/build.sh + python3 -m pip install pandas pyarrow bash -x ./chdb/test_smoke.sh continue-on-error: false - name: Check ccache statistics @@ -193,6 +194,7 @@ jobs: export CXX=$(brew --prefix llvm@15)/bin/clang++ bash gen_manifest.sh bash ./chdb/build.sh + python3 -m pip install pandas pyarrow bash -x ./chdb/test_smoke.sh continue-on-error: false - name: Keep killall ccache and wait for ccache to finish diff --git a/chdb/__init__.py b/chdb/__init__.py index 9b8c83929e7..f5167cc5ba9 100644 --- a/chdb/__init__.py +++ b/chdb/__init__.py @@ -22,20 +22,32 @@ except: # pragma: no cover __version__ = "unknown" - -def _to_arrowTable(res): +# return pyarrow table +def to_arrowTable(res): """convert res to arrow table""" - import pyarrow as pa #import when needed + # try import pyarrow and pandas, if failed, raise ImportError with suggestion + try: + import pyarrow as pa + import pandas + except ImportError as e: + print(f'ImportError: {e}') + print('Please install pyarrow and pandas via "pip install pyarrow pandas"') + raise ImportError('Failed to import pyarrow or pandas') from None + return pa.RecordBatchFileReader(res.get_memview()).read_all() +# return pandas dataframe def to_df(r): """"convert arrow table to Dataframe""" - t = _to_arrowTable(r) + t = to_arrowTable(r) return t.to_pandas(use_threads=True) # wrap _chdb functions def query(sql, output_format="CSV", **kwargs): - if output_format.lower() == "dataframe": - r = _chdb.query(sql, "Arrow", **kwargs) - return to_df(r) - return _chdb.query(sql, output_format, **kwargs) + lower_output_format = output_format.lower() + if lower_output_format == "dataframe": + return to_df(_chdb.query(sql, "Arrow", **kwargs)) + elif lower_output_format == 'arrowtable': + return to_arrowTable(_chdb.query(sql, "Arrow", **kwargs)) + else: + return _chdb.query(sql, output_format, **kwargs) diff --git a/setup.py b/setup.py index fd17a538c72..5a0e57e66e0 100644 --- a/setup.py +++ b/setup.py @@ -173,7 +173,6 @@ def build_extensions(self): exclude_package_data={'': ['*.pyc', 'src/**']}, ext_modules=ext_modules, python_requires='>=3.7', - install_requires=['pyarrow', 'pandas'], cmdclass={'build_ext': BuildExt}, test_suite="tests", zip_safe=False, From f8aefc23af8f331c9de16ed40a44ee03e222416f Mon Sep 17 00:00:00 2001 From: auxten Date: Mon, 17 Apr 2023 18:46:15 +0800 Subject: [PATCH 4/6] Build on pr based on pybind and push tags --- .github/workflows/build_wheels.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 6acf8c38e08..f24c21c15b2 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -1,6 +1,12 @@ name: Build -on: [push, pull_request] +on: + push: + tags: + - 'v*' + pull_request: + branches: + - pybind jobs: build_wheels_linux: From 21146ea87a5c1a6a0118182d82e74363da0fd5f6 Mon Sep 17 00:00:00 2001 From: auxten Date: Mon, 17 Apr 2023 22:27:23 +0800 Subject: [PATCH 5/6] Sleep 60 then kill clang if timeout --- .github/workflows/build_wheels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index f24c21c15b2..37cf5fb9aeb 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -206,7 +206,7 @@ jobs: - name: Keep killall ccache and wait for ccache to finish if: always() run: | - sleep 300 + sleep 60 while ps -ef | grep ccache | grep -v grep; do \ killall ccache; \ sleep 10; \ @@ -357,7 +357,7 @@ jobs: - name: Keep killall ccache and wait for ccache to finish if: always() run: | - sleep 300 + sleep 60 while ps -ef | grep ccache | grep -v grep; do \ killall ccache; \ sleep 10; \ From cd14bf713f1092d22488dfcc79e6c2873da37e39 Mon Sep 17 00:00:00 2001 From: auxten Date: Tue, 18 Apr 2023 16:16:47 +0800 Subject: [PATCH 6/6] Update brew to avoid llvm@15 minor version unstable --- .github/workflows/build_wheels.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml index 37cf5fb9aeb..ad7b3940c30 100644 --- a/.github/workflows/build_wheels.yml +++ b/.github/workflows/build_wheels.yml @@ -155,7 +155,6 @@ jobs: run: | pwd uname -a - export HOMEBREW_NO_AUTO_UPDATE=1 export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 brew install git ccache ninja libtool gettext llvm@15 gcc binutils grep findutils zstd export PATH=$(brew --prefix llvm@15)/bin:$PATH @@ -269,7 +268,6 @@ jobs: run: | pwd uname -a - export HOMEBREW_NO_AUTO_UPDATE=1 export HOMEBREW_NO_INSTALLED_DEPENDENTS_CHECK=1 brew install git ccache ninja libtool gettext llvm@15 gcc binutils grep findutils zstd export PATH=$(brew --prefix llvm@15)/bin:$PATH