Skip to content

Add support for {DataFrame,Series}.mode #6351

Add support for {DataFrame,Series}.mode

Add support for {DataFrame,Series}.mode #6351

Workflow file for this run

name: Mars CI on Platforms
on:
push:
branches:
- '*'
pull_request:
types: ['opened', 'reopened', 'synchronize']
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest]
python-version: [3.8-kubernetes, 3.8-ray, 3.8-ray-deploy, 3.8-ray-dag, 3.8-vineyard, 3.8-dask]
include:
- { os: ubuntu-latest, python-version: 3.8-kubernetes, no-common-tests: 1,
no-deploy: 1, with-kubernetes: "with Kubernetes" }
- { os: ubuntu-20.04, python-version: 3.8-hadoop, no-common-tests: 1,
no-deploy: 1, with-hadoop: "with hadoop" }
- { os: ubuntu-latest, python-version: 3.8-vineyard, no-common-tests: 1,
no-deploy: 1, with-vineyard: "with vineyard" }
- { os: ubuntu-latest, python-version: 3.8-ray, no-common-tests: 1,
no-deploy: 1, with-ray: "with ray" }
- { os: ubuntu-latest, python-version: 3.8-ray-deploy, no-common-tests: 1,
no-deploy: 1, with-ray-deploy: "with ray deploy" }
- { os: ubuntu-latest, python-version: 3.8-ray-dag, no-common-tests: 1,
no-deploy: 1, with-ray-dag: "with ray dag" }
- { os: ubuntu-latest, python-version: 3.8-dask, no-common-tests: 1,
no-deploy: 1, run-dask: "run dask" }
steps:
- name: Check out code
uses: actions/checkout@v2
with:
fetch-depth: 2
- name: Set up conda ${{ matrix.python-version }}
env:
PYTHON: ${{ matrix.python-version }}
shell: bash
run: |
source ./ci/install-conda.sh
python -m pip install --upgrade pip setuptools wheel coverage
- name: Start minikube
if: ${{ matrix.with-kubernetes }}
uses: medyagh/setup-minikube@master
with:
driver: docker
mount-path: '/home/runner:/home/runner'
- name: Install dependencies
env:
WITH_HADOOP: ${{ matrix.with-hadoop }}
WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
WITH_VINEYARD: ${{ matrix.with-vineyard }}
WITH_RAY: ${{ matrix.with-ray }}
WITH_RAY_DEPLOY: ${{ matrix.with-ray-deploy }}
WITH_RAY_DAG: ${{ matrix.with-ray-dag }}
RUN_DASK: ${{ matrix.run-dask }}
NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
shell: bash
run: |
source ./ci/reload-env.sh
export DEFAULT_VENV=$VIRTUAL_ENV
source ./ci/rewrite-cov-config.sh
pip install numpy scipy cython
pip install -e ".[dev,extra]"
if [[ $UNAME == "windows" ]]; then
pip install virtualenv flaky
else
pip install virtualenv flaky
if [ -n "$WITH_KUBERNETES" ]; then
pip install kubernetes "pandas<1.5.0"
kubectl get pods -A
fi
if [ -n "$WITH_HADOOP" ]; then
./ci/install-hadoop.sh
echo "import coverage; coverage.process_startup()" > \
$(python -c "import site; print(site.getsitepackages()[-1])")/coverage.pth
conda install -n test --quiet --yes -c conda-forge python=$PYTHON skein libffi conda-pack
fi
if [ -n "$WITH_VINEYARD" ]; then
pip install vineyard -i https://pypi.org/simple
mkdir -p /tmp/etcd-download-test
export ETCD_VER=v3.4.13
export ETCD_DOWNLOAD_URL=https://github.com/etcd-io/etcd/releases/download
curl -L $ETCD_DOWNLOAD_URL/$ETCD_VER/etcd-$ETCD_VER-linux-amd64.tar.gz -o /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz
tar xzvf /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz -C /tmp/etcd-download-test --strip-components=1
sudo mv /tmp/etcd-download-test/etcd /usr/local/bin/
sudo mv /tmp/etcd-download-test/etcdctl /usr/local/bin/
rm -fr /tmp/etcd-$ETCD_VER-linux-amd64.tar.gz /tmp/etcd-download-test
fi
if [ -n "$WITH_RAY" ] || [ -n "$WITH_RAY_DAG" ] || [ -n "$WITH_RAY_DEPLOY" ]; then
pip install "ray>=1.8.0,<2.4.0"
pip install "xgboost_ray<0.1.14" "protobuf<4"
# Ray Datasets need pyarrow>=6.0.1
pip install "pyarrow>=6.0.1"
pip install lightgbm
fi
if [ -n "$RUN_DASK" ]; then
pip install "dask[complete]" "mimesis<9.0.0" scikit-learn
fi
fi
conda list -n test
- name: Test with pytest
env:
WITH_HADOOP: ${{ matrix.with-hadoop }}
WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
WITH_CYTHON: ${{ matrix.with-cython }}
WITH_VINEYARD: ${{ matrix.with-vineyard }}
WITH_RAY: ${{ matrix.with-ray }}
WITH_RAY_DEPLOY: ${{ matrix.with-ray-deploy }}
WITH_RAY_DAG: ${{ matrix.with-ray-dag }}
RUN_DASK: ${{ matrix.run-dask }}
NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
NUMPY_EXPERIMENTAL_ARRAY_FUNCTION: 1
USE_MINIKUBE_DOCKER_ENV: true
CHANGE_MINIKUBE_NONE_USER: true
shell: bash
run: |
source ./ci/reload-env.sh
if [ -n "$WITH_HADOOP" ]; then
source $CONDA/bin/activate test
source ./ci/reload-env.sh
pytest $PYTEST_CONFIG -m hadoop
coverage report
fi
if [ -n "$WITH_KUBERNETES" ]; then
pytest $PYTEST_CONFIG --forked mars/deploy/kubernetes
coverage report
fi
if [ -n "$WITH_VINEYARD" ]; then
pytest $PYTEST_CONFIG mars/storage/tests/test_libs.py
mv .coverage build/.coverage.test_lib.file
pytest $PYTEST_CONFIG mars/deploy/oscar/tests/test_local.py
mv .coverage build/.coverage.test_local.file
pytest $PYTEST_CONFIG -k "vineyard" \
mars/tensor/datastore/tests/test_datastore_execution.py \
mars/dataframe/datastore/tests/test_datastore_execution.py
mv .coverage build/.coverage.test_vineyard_op.file
coverage combine build/ && coverage report
fi
if [ -n "$WITH_RAY" ]; then
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s --ignore=mars/deploy/oscar/ -m ray
coverage report
fi
if [ -n "$WITH_RAY_DEPLOY" ]; then
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray.py -m ray
mv .coverage build/.coverage.test_ray.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_load_modules.py -m ray
mv .coverage build/.coverage.test_ray_load_modules.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_cluster_standalone.py -m ray
mv .coverage build/.coverage.test_ray_cluster_standalone.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_client.py -m ray
mv .coverage build/.coverage.test_ray_client.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_fault_injection.py -m ray
mv .coverage build/.coverage.test_ray_fault_injection.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_scheduling.py -m ray
mv .coverage build/.coverage.test_ray_scheduling.file
coverage combine build/ && coverage report
fi
if [ -n "$WITH_RAY_DAG" ]; then
export MARS_CI_BACKEND=ray
export RAY_idle_worker_killing_time_threshold_ms=60000
pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/dataframe -v -s -m "not skip_ray_dag" --ignore=mars/dataframe/contrib/raydataset
pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/dataframe/contrib/raydataset -v -s -m "not skip_ray_dag"
pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/tensor -v -s -m "not skip_ray_dag"
pytest $PYTEST_CONFIG --durations=0 --timeout=500 mars/learn --ignore mars/learn/contrib --ignore mars/learn/utils/tests/test_collect_ports.py -v -s -m "not skip_ray_dag"
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s -m ray_dag
mv .coverage build/.coverage.ray_dag.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_dag.py
mv .coverage build/.coverage.test_ray_dag.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_dag_failover.py
mv .coverage build/.coverage.test_ray_dag_failover.file
pytest $PYTEST_CONFIG --durations=0 --timeout=200 -v -s mars/deploy/oscar/tests/test_ray_dag_oscar.py -m ray
mv .coverage build/.coverage.test_ray_dag_oscar.file
coverage combine build/ && coverage report
fi
if [ -n "$RUN_DASK" ]; then
pytest $PYTEST_CONFIG mars/contrib/dask/tests/test_dask.py
coverage report
fi
coverage xml
- name: Stop vineyard runtime
if: ${{ matrix.with-vineyard }}
shell: bash
run: |
sudo docker stop vineyard || true
- name: Report coverage data
shell: bash
run: |
bash <(curl -s https://codecov.io/bash)