Add cibuildwheel with Linux AArch64 wheel build support (#2672) (#2854)

.github/workflows/pypi-cd.yml

@@ -7,63 +7,55 @@ on:
 
 jobs:
   build:
+    name: Build wheels on ${{ matrix.os }} for ${{ matrix.arch }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macOS-latest, windows-latest]
-        python-version: [3.6, 3.7, 3.8, 3.9]
-        node-version: [14.x]
-        include:
-          - { os: ubuntu-latest, python-version: 3.6, python-abis: "cp36-cp36m" }
-          - { os: ubuntu-latest, python-version: 3.7, python-abis: "cp37-cp37m" }
-          - { os: ubuntu-latest, python-version: 3.8, python-abis: "cp38-cp38" }
-          - { os: ubuntu-latest, python-version: 3.9, python-abis: "cp39-cp39" }
-          - { os: windows-latest, python-version: 3.9, build-static: 1 }
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        arch: [auto]
 
     steps:
       - name: Check out code
         uses: actions/checkout@v2
 
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
-        with:
-          node-version: ${{ matrix.node-version }}
-
-      - name: Set up conda ${{ matrix.python-version }}
+      - name: Build wheels
+        uses: pypa/[email protected]
         env:
-          PYTHON: ${{ matrix.python-version }}
-        shell: bash
-        run: |
-          source ./ci/install-conda.sh
-          python -m pip install --upgrade pip setuptools wheel coverage;
+          CIBW_BEFORE_BUILD: git reset --hard && pip install -r ci/requirements-wheel.txt
+          CIBW_ARCHS: ${{ matrix.arch }}
+          CIBW_SKIP: pp* *-musllinux* cp310-* *i686
+          CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
 
-      - name: Install dependencies
-        env:
-          WITH_HADOOP: ${{ matrix.with-hadoop }}
-          WITH_KUBERNETES: ${{ matrix.with-kubernetes }}
-          NO_COMMON_TESTS: ${{ matrix.no-common-tests }}
-        shell: bash
+      - name: Build source
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.arch == 'auto'}}
         run: |
-          source ./ci/reload-env.sh
-          export DEFAULT_VENV=$VIRTUAL_ENV
+          git reset --hard
+          pip install -r ci/requirements-wheel.txt
+          python setup.py sdist --formats=gztar --dist-dir=./wheelhouse
 
-          if [[ "$PYTHON" =~ "3.9" ]]; then
-            conda install -n test --quiet --yes -c conda-forge python=$PYTHON pyarrow
-          fi
-
-          pip install numpy scipy cython
-          pip install -e ".[dev,extra]"
-          conda list -n test
-
-      - name: Deploy packages
-        if: startsWith(github.ref, 'refs/tags/') && matrix.no-deploy != '1'
+      - name: Release to pypi
         shell: bash
         env:
-          DOCKER_IMAGE: "quay.io/pypa/manylinux1_x86_64"
-          PYABI: ${{ matrix.python-abis }}
-          BUILD_STATIC: ${{ matrix.build-static }}
           PYPI_PWD: ${{ secrets.PYPI_PASSWORD }}
         run: |
-          source ./ci/reload-env.sh
-          source ./.github/workflows/upload-packages.sh
+          if [[ "$GITHUB_REPOSITORY" == "mars-project/mars" ]]; then
+            PYPI_REPO="https://upload.pypi.org/legacy/"
+          else
+            PYPI_REPO="https://test.pypi.org/legacy/"
+          fi
+          echo "[distutils]"             > ~/.pypirc
+          echo "index-servers ="        >> ~/.pypirc
+          echo "    pypi"               >> ~/.pypirc
+          echo "[pypi]"                 >> ~/.pypirc
+          echo "repository=$PYPI_REPO"  >> ~/.pypirc
+          echo "username=pyodps"        >> ~/.pypirc
+          echo "password=$PYPI_PWD"     >> ~/.pypirc
+          python -m pip install twine
+          python -m twine upload -r pypi --skip-existing wheelhouse/*
+
+      - name: Upload artifacts to github
+        uses: actions/upload-artifact@v1
+        with:
+          name: wheels
+          path: ./wheelhouse

.gitignore

@@ -10,6 +10,7 @@
 *.egg
 *.egg-info
 dist
+wheelhouse
 build/
 eggs
 parts

ci/requirements-wheel.txt

@@ -1,9 +1,13 @@
-numpy==1.14.5; python_version<'3.9'
-numpy==1.19.3; python_version>='3.9'
-pandas==1.0.4; python_version<'3.9'
-pandas==1.1.3; python_version>='3.9'
-scipy==1.4.1; python_version<'3.9'
-scipy==1.5.4; python_version>='3.9'
-cython==0.29.21
+oldest-supported-numpy
+
+pandas==1.0.4; python_version<'3.9' and platform_machine!='aarch64'
+pandas==1.1.3; python_version<'3.9' and platform_machine=='aarch64'
+pandas==1.2.2; python_version>='3.9' and python_version<'3.10'
+
+scipy==1.4.1; python_version<'3.9' and platform_machine!='aarch64'
+scipy==1.5.3; python_version<'3.9' and platform_machine=='aarch64'
+scipy==1.5.4; python_version>='3.9' and python_version<'3.10'
+
+cython==0.29.26
 requests>=2.4.0
 cloudpickle>=1.5.0

mars/serialization/numpy.py

@@ -61,7 +61,12 @@ def deserialize(self, header: Dict, buffers: List, context: Dict):
         if header["pickle"]:
             return unpickle_buffers(buffers)
 
-        dtype = np.lib.format.descr_to_dtype(header["descr"])
+        try:
+            dtype = np.lib.format.descr_to_dtype(header["descr"])
+        except AttributeError:  # pragma: no cover
+            # for older numpy versions, descr_to_dtype is not implemented
+            dtype = np.dtype(header["descr"])
+
         dtype_new_order = header["dtype_new_order"]
         if dtype_new_order:
             dtype = dtype[dtype_new_order]