Fix conversion from arrow column to arrow RecordBatch

mars/dataframe/arrays.py

@@ -301,6 +301,12 @@ def __repr__(self):
     def _array(self):
         return self._arrow_array if self._use_arrow else self._ndarray
 
+    def __arrow_array__(self, type=None):
+        if self._use_arrow:
+            combined = self._arrow_array.combine_chunks()
+            return combined.cast(type) if type else combined
+        return super().__arrow_array__(type=type)
+
     @property
     def dtype(self) -> "Type[ArrowDtype]":
         return self._dtype

mars/dataframe/core.py

@@ -67,6 +67,7 @@
     tokenize,
     estimate_pandas_size,
     calc_nsplits,
+    is_debugger_repr_thread,
 )
 from .utils import fetch_corner_data, ReprSeries, parse_index, merge_index_value
 
@@ -1430,6 +1431,10 @@ def __mars_tensor__(self, dtype=None, order="K"):
         return tensor.astype(dtype=dtype, order=order, copy=False)
 
     def iteritems(self, batch_size=10000, session=None):
+        if is_build_mode():
+            raise NotImplementedError("Not implemented when building dags")
+        if is_debugger_repr_thread() and len(self._executed_sessions) == 0:
+            raise NotImplementedError("Not implemented when not executed under debug")
         for batch_data in self.iterbatch(batch_size=batch_size, session=session):
             yield from getattr(batch_data, "iteritems")()
 

mars/dataframe/tests/test_arrays.py

@@ -480,3 +480,7 @@ def test_to_pandas():
     s2 = df2["b"].str[:2]
     expected = df["b"].astype("string").str[:2]
     pd.testing.assert_series_equal(s2, expected)
+
+    # test reverse conversion to arrow
+    arrow_data = pa.RecordBatch.from_pandas(df2)
+    assert arrow_data.num_rows == len(df2)

mars/learn/contrib/lightgbm/classifier.py

@@ -24,8 +24,9 @@
     lightgbm = None
 
 
-LGBMClassifier = make_import_error_func("lightgbm")
-if lightgbm:
+if not lightgbm:
+    LGBMClassifier = make_import_error_func("lightgbm")
+else:
 
     class LGBMClassifier(LGBMScikitLearnBase, lightgbm.LGBMClassifier):
         def fit(
@@ -42,7 +43,10 @@ def fit(
             **kwargs
         ):
             check_consistent_length(X, y, session=session, run_kwargs=run_kwargs)
+
             params = self.get_params(True)
+            self._fix_verbose_args(kwargs, params)
+
             model = train(
                 params,
                 self._wrap_train_tuple(X, y, sample_weight, init_score),

mars/learn/contrib/lightgbm/core.py

@@ -19,8 +19,9 @@
 import numpy as np
 import pandas as pd
 
-from ....tensor import tensor as mars_tensor
 from ....dataframe import DataFrame as MarsDataFrame, Series as MarsSeries
+from ....lib.version import parse as parse_version
+from ....tensor import tensor as mars_tensor
 
 
 class LGBMModelType(enum.Enum):
@@ -58,6 +59,16 @@ def __init__(self, *args, **kwargs):
         else:
             super().__init__(*args, **kwargs)
 
+    @classmethod
+    def _fix_verbose_args(cls, kwds, params):
+        if "verbose" not in kwds:
+            return
+
+        import lightgbm
+
+        if parse_version(lightgbm.__version__).major >= 4:
+            params["verbose"] = kwds.pop("verbose")
+
     @classmethod
     def _get_lgbm_class(cls):
         try:

mars/learn/contrib/lightgbm/ranker.py

@@ -24,8 +24,9 @@
     lightgbm = None
 
 
-LGBMRanker = make_import_error_func("lightgbm")
-if lightgbm:
+if not lightgbm:
+    LGBMRanker = make_import_error_func("lightgbm")
+else:
 
     class LGBMRanker(LGBMScikitLearnBase, lightgbm.LGBMRanker):
         def fit(
@@ -43,7 +44,10 @@ def fit(
             **kwargs
         ):
             check_consistent_length(X, y, session=session, run_kwargs=run_kwargs)
+
             params = self.get_params(True)
+            self._fix_verbose_args(kwargs, params)
+
             model = train(
                 params,
                 self._wrap_train_tuple(X, y, sample_weight, init_score),

mars/learn/contrib/lightgbm/regressor.py

@@ -24,8 +24,9 @@
     lightgbm = None
 
 
-LGBMRegressor = make_import_error_func("lightgbm")
-if lightgbm:
+if not lightgbm:
+    LGBMRegressor = make_import_error_func("lightgbm")
+else:
 
     class LGBMRegressor(LGBMScikitLearnBase, lightgbm.LGBMRegressor):
         def fit(
@@ -42,7 +43,10 @@ def fit(
             **kwargs
         ):
             check_consistent_length(X, y, session=session, run_kwargs=run_kwargs)
+
             params = self.get_params(True)
+            self._fix_verbose_args(kwargs, params)
+
             model = train(
                 params,
                 self._wrap_train_tuple(X, y, sample_weight, init_score),

mars/learn/contrib/xgboost/classifier.py

@@ -16,8 +16,9 @@
 from .core import xgboost, XGBScikitLearnBase
 
 
-XGBClassifier = make_import_error_func("xgboost")
-if xgboost:
+if not xgboost:
+    XGBClassifier = make_import_error_func("xgboost")
+else:
     from xgboost.sklearn import XGBClassifierBase
 
     from .... import tensor as mt

mars/learn/contrib/xgboost/core.py

@@ -22,8 +22,9 @@
 from .dmatrix import MarsDMatrix
 
 
-XGBScikitLearnBase = None
-if xgboost:
+if not xgboost:
+    XGBScikitLearnBase = None
+else:
 
     class XGBScikitLearnBase(xgboost.XGBModel):
         """

mars/learn/contrib/xgboost/regressor.py

@@ -17,8 +17,9 @@
 from .core import xgboost, XGBScikitLearnBase
 
 
-XGBRegressor = make_import_error_func("xgboost")
-if xgboost:
+if not xgboost:
+    XGBRegressor = make_import_error_func("xgboost")
+else:
     from .core import wrap_evaluation_matrices
     from .train import train
     from .predict import predict

mars/lib/mkl_interface.py

@@ -46,12 +46,12 @@ class MKLVersion(ctypes.Structure):
     ]
 
 
-mkl_free_buffers = None
-mkl_get_version = None
-mkl_mem_stat = None
-
 mkl_rt = _load_mkl_rt("mkl_rt")
-if mkl_rt:
+if not mkl_rt:
+    mkl_free_buffers = None
+    mkl_get_version = None
+    mkl_mem_stat = None
+else:
     try:
         mkl_free_buffers = mkl_rt.mkl_free_buffers
         mkl_free_buffers.argtypes = []

mars/utils.py

@@ -1673,7 +1673,7 @@ def get_func_token(func):
 
 
 def _get_func_token_values(func):
-    if hasattr(func, "__code__"):
+    if hasattr(func, "__code__") and func.__code__.co_code:
         tokens = [func.__code__.co_code]
         if func.__closure__ is not None:
             cvars = tuple([x.cell_contents for x in func.__closure__])
@@ -1684,10 +1684,13 @@ def _get_func_token_values(func):
         while isinstance(func, functools.partial):
             tokens.extend([func.args, func.keywords])
             func = func.func
-        if hasattr(func, "__code__"):
-            tokens.extend(_get_func_token_values(func))
-        elif isinstance(func, types.BuiltinFunctionType):
+        if (
+            isinstance(func, types.BuiltinFunctionType)
+            or "cython" in type(func).__name__
+        ):
             tokens.extend([func.__module__, func.__name__])
+        elif hasattr(func, "__code__"):
+            tokens.extend(_get_func_token_values(func))
         else:
             tokens.append(func)
         return tokens
@@ -1912,3 +1915,8 @@ def get_node_ip_address(address="8.8.8.8:53"):
         s.close()
 
     return node_ip_address
+
+
+def is_debugger_repr_thread():
+    thread_cls_name = type(threading.current_thread()).__name__
+    return "GetValue" in thread_cls_name and "Debug" in thread_cls_name

setup.cfg

@@ -30,7 +30,7 @@ install_requires =
     pandas>=1.0.0,<2.0.0
     scipy>=1.0.0
     scikit-learn>=0.20
-    numexpr>=2.6.4
+    numexpr>=2.6.4,!=2.8.5
     cloudpickle>=1.5.0
     pyyaml>=5.1
     psutil>=5.9.0

setup.py

@@ -20,11 +20,10 @@
 import warnings
 from sysconfig import get_config_vars
 
-from pkg_resources import parse_version
-from setuptools import setup, Extension, Command
-
 import numpy as np
 from Cython.Build import cythonize
+from pkg_resources import parse_version
+from setuptools import Command, Extension, setup
 from setuptools.command.develop import develop
 from setuptools.command.install import install
 from setuptools.command.sdist import sdist

versioneer.py

@@ -282,13 +282,13 @@
 
 import configparser
 import errno
+import functools
 import json
 import os
 import re
 import subprocess
 import sys
 from typing import Callable, Dict
-import functools
 
 
 class VersioneerConfig:
@@ -1872,6 +1872,7 @@ def run(self):
 
     if "cx_Freeze" in sys.modules:  # cx_freeze enabled?
         from cx_Freeze.dist import build_exe as _build_exe
+
         # nczeczulin reports that py2exe won't like the pep440-style string
         # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g.
         # setup(console=[{