diff --git a/.github/workflows/build-wheel-linux-x86_64.yaml b/.github/workflows/build-wheel-linux-x86_64.yaml index 1c9c236bc0..650e63432d 100644 --- a/.github/workflows/build-wheel-linux-x86_64.yaml +++ b/.github/workflows/build-wheel-linux-x86_64.yaml @@ -119,32 +119,48 @@ jobs: run: | # Reduce wait time for repos not responding cat /etc/yum.conf | sed "s/\[main\]/\[main\]\ntimeout=5/g" > /etc/yum.conf - yum update -y && yum install -y cmake ninja-build + yum update -y && yum install -y cmake ninja-build libzstd-devel - name: Install Dependencies (Python) run: | python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML + # Required for MHLO and building MLIR with protected symbols. + # (Don't forget to add the build directory to PATH in subsequent steps, so + # other tools can find it, in particular collect2 invoked by gcc.) + - name: Build LLD + if: steps.cache-llvm-build.outputs.cache-hit != 'true' + run: | + cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLLVM_TARGETS_TO_BUILD="host" \ + -DLLVM_ENABLE_PROJECTS="lld" + + cmake --build llvm-build --target lld + - name: Build LLVM / MLIR if: steps.cache-llvm-build.outputs.cache-hit != 'true' run: | + export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_BUILD_EXAMPLES=OFF \ -DLLVM_TARGETS_TO_BUILD="host" \ - -DLLVM_ENABLE_PROJECTS="lld;mlir" \ + -DLLVM_ENABLE_PROJECTS="mlir" \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_INSTALL_UTILS=ON \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_ZSTD=FORCE_ON \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ - -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") + -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ + -DCMAKE_CXX_VISIBILITY_PRESET=protected \ + -DLLVM_ENABLE_LLD=ON # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing # and remove filter # This tests fails on CI/CD not locally. - LIT_FILTER_OUT="Bytecode" cmake --build llvm-build --target lld check-mlir + LIT_FILTER_OUT="Bytecode" cmake --build llvm-build --target check-mlir - name: Build MHLO Dialect if: steps.cache-mhlo-build.outputs.cache-hit != 'true' @@ -155,19 +171,23 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ - -DLLVM_ENABLE_LLD=ON \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF + -DLLVM_ENABLE_ZSTD=FORCE_ON \ + -DCMAKE_CXX_VISIBILITY_PRESET=protected \ + -DLLVM_ENABLE_LLD=ON cmake --build mhlo-build --target check-mlir-hlo - name: Build Enzyme if: steps.cache-enzyme-build.outputs.cache-hit != 'true' run: | + export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH cmake -S mlir/Enzyme/enzyme -B enzyme-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm \ - -DENZYME_STATIC_LIB=ON + -DENZYME_STATIC_LIB=ON \ + -DCMAKE_CXX_VISIBILITY_PRESET=protected \ + -DLLVM_ENABLE_LLD=ON cmake --build enzyme-build --target EnzymeStatic-18 @@ -193,11 +213,11 @@ jobs: run: | # Reduce wait time for repos not responding cat /etc/yum.conf | sed "s/\[main\]/\[main\]\ntimeout=5/g" > /etc/yum.conf - yum update -y && yum install -y cmake ninja-build openmpi-devel + yum update -y && yum install -y cmake ninja-build openmpi-devel libzstd-devel - name: Install Dependencies (Python) run: | - python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML scipy + python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML - name: Get Cached LLVM Source id: cache-llvm-source @@ -263,13 +283,15 @@ jobs: -DKokkos_ENABLE_OPENMP=ON \ -DENABLE_WARNINGS=OFF \ -DENABLE_OPENQASM=ON \ - -DENABLE_OPENMP=ON + -DENABLE_OPENMP=OFF \ + -DLQ_ENABLE_KERNEL_OMP=OFF cmake --build runtime-build --target rt_capi rtd_lightning rtd_openqasm # Build Quantum and Gradient Dialects - name: Build MLIR Dialects run: | + export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH cmake -S mlir -B quantum-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ @@ -282,7 +304,8 @@ jobs: -DEnzyme_DIR=$GITHUB_WORKSPACE/enzyme-build \ -DENZYME_SRC_DIR=$GITHUB_WORKSPACE/mlir/Enzyme \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF + -DLLVM_ENABLE_ZSTD=FORCE_ON \ + -DLLVM_ENABLE_LLD=ON cmake --build quantum-build --target check-dialects compiler_driver @@ -296,11 +319,16 @@ jobs: ENZYME_BUILD_DIR=$GITHUB_WORKSPACE/enzyme-build \ make wheel + - name: Repair wheel using auditwheel + run: | + # exclude libopenblas as we rely on the openblas/lapack library shipped by scipy + auditwheel repair dist/*.whl -w ./wheel --no-update-tags --exclude libopenblasp-r0-23e5df77.3.21.dev.so + - name: Upload Wheel Artifact uses: actions/upload-artifact@v3 with: name: catalyst-manylinux2014_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: dist/ + path: wheel/ retention-days: 14 test-wheels: diff --git a/.github/workflows/build-wheel-macos-arm64.yaml b/.github/workflows/build-wheel-macos-arm64.yaml index 0c34a08efd..1550cc7a8b 100644 --- a/.github/workflows/build-wheel-macos-arm64.yaml +++ b/.github/workflows/build-wheel-macos-arm64.yaml @@ -6,7 +6,7 @@ on: workflow_dispatch: env: - MACOSX_DEPLOYMENT_TARGET: 13.0 + MACOSX_DEPLOYMENT_TARGET: 14.0 concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -23,23 +23,28 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest-xlarge] - arch: [arm64] python_version: ${{ fromJson(needs.constants.outputs.python_versions) }} name: Build Dependencies (Python ${{ matrix.python_version }}) - runs-on: macos-latest-xlarge + runs-on: + group: 'Office 24th floor M2 Mac' steps: - name: Checkout Catalyst repo uses: actions/checkout@v3 + - name: Setup Runner Environment + id: setup_env + uses: ./.github/workflows/utils/setup_self_hosted_macos_env + with: + python_version: ${{ matrix.python_version }} + # Cache external project sources - name: Cache LLVM Source id: cache-llvm-source uses: actions/cache@v3 with: - path: mlir/llvm-project + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project key: Linux-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source enableCrossOsArchive: True @@ -47,7 +52,7 @@ jobs: id: cache-mhlo-source uses: actions/cache@v3 with: - path: mlir/mlir-hlo + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo key: Linux-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source enableCrossOsArchive: True @@ -55,7 +60,7 @@ jobs: id: cache-enzyme-source uses: actions/cache@v3 with: - path: mlir/Enzyme + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme key: Linux-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source enableCrossOsArchive: True @@ -83,40 +88,64 @@ jobs: ref: ${{ needs.constants.outputs.enzyme_version }} path: mlir/Enzyme + # This step is needed because actions/checkout cannot clone to outside GITHUB_WORKSPACE + # https://github.com/actions/checkout/issues/197 + - name: Copy Submodule to tmp cache directory + run: | + if [ ! -d "${{ steps.setup_env.outputs.dependency_build_dir }}/mlir" ]; then + echo 'Creating directory at ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir' + mkdir -p ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir + fi + + if [ "${{ steps.cache-llvm-source.outputs.cache-hit }}" != 'true' ]; then + echo 'Copying mlir/llvm-project to ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/' + mkdir -p ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project + mv mlir/llvm-project/* ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project + fi + + if [ "${{ steps.cache-mhlo-source.outputs.cache-hit }}" != 'true' ]; then + echo 'Copying mlir/mlir-hlo to ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/' + mkdir -p ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo + mv mlir/mlir-hlo/* ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo + fi + + if [ "${{ steps.cache-enzyme-source.outputs.cache-hit }}" != 'true' ]; then + echo 'Copying mlir/Enzyme to ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/' + mkdir -p ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme + mv mlir/Enzyme/* ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme + fi + + ls ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/ + # Cache external project builds - name: Cache LLVM Build id: cache-llvm-build uses: actions/cache@v3 with: - path: llvm-build + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-generic-build - name: Cache MHLO Build id: cache-mhlo-build uses: actions/cache@v3 with: - path: mhlo-build + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build - name: Cache Enzyme Build id: cache-enzyme-build uses: actions/cache@v3 with: - path: enzyme-build + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build - - name: Set up Python via brew - run: | - brew install python@${{ matrix.python_version }} - - name: Install Dependencies (Python) - run: | - python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML cmake ninja + run: python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML cmake ninja - name: Build LLVM / MLIR if: steps.cache-llvm-build.outputs.cache-hit != 'true' run: | - cmake -S mlir/llvm-project/llvm -B llvm-build -G Ninja \ + cmake -S ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project/llvm -B ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_BUILD_EXAMPLES=OFF \ -DLLVM_TARGETS_TO_BUILD="host" \ @@ -124,70 +153,74 @@ jobs: -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_INSTALL_UTILS=ON \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_ZSTD=FORCE_ON \ -DLLVM_ENABLE_LLD=OFF \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ - -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") + -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing # and remove filter # This tests fails on CI/CD not locally. - LIT_FILTER_OUT="Bytecode" cmake --build llvm-build --target check-mlir + LIT_FILTER_OUT="Bytecode" cmake --build ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build --target check-mlir - name: Build MHLO Dialect if: steps.cache-mhlo-build.outputs.cache-hit != 'true' run: | - export PATH=$GITHUB_WORKSPACE/llvm-build/bin:$PATH - cmake -S mlir/mlir-hlo -B mhlo-build -G Ninja \ + export PATH=${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build/bin:$PATH + cmake -S ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo -B ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_ASSERTIONS=ON \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ + -DMLIR_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build/lib/cmake/mlir \ -DLLVM_ENABLE_LLD=OFF \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF + -DLLVM_ENABLE_ZSTD=FORCE_ON \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden - cmake --build mhlo-build --target check-mlir-hlo + cmake --build ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build --target check-mlir-hlo - name: Build Enzyme if: steps.cache-enzyme-build.outputs.cache-hit != 'true' run: | - cmake -S mlir/Enzyme/enzyme -B enzyme-build -G Ninja \ + cmake -S ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme/enzyme -B ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm \ - -DENZYME_STATIC_LIB=ON + -DLLVM_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build/lib/cmake/llvm \ + -DENZYME_STATIC_LIB=ON \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden - cmake --build enzyme-build --target EnzymeStatic-18 + cmake --build ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build --target EnzymeStatic-18 catalyst-macos-wheels-arm64: needs: [constants, build-dependencies] strategy: fail-fast: false matrix: - os: [macos-latest-xlarge] - arch: [arm64] python_version: ${{ fromJson(needs.constants.outputs.python_versions) }} name: Build Wheels (Python ${{ matrix.python_version }}) - runs-on: ${{ matrix.os }} + runs-on: + group: 'Office 24th floor M2 Mac' steps: - name: Checkout Catalyst repo uses: actions/checkout@v3 - - name: Set up Python via brew - run: | - brew install python@${{ matrix.python_version }} + - name: Setup Runner Environment + id: setup_env + uses: ./.github/workflows/utils/setup_self_hosted_macos_env + with: + python_version: ${{ matrix.python_version }} - name: Install Dependencies (Python) run: | - python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML cmake ninja scipy + python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML cmake ninja delocate - name: Get Cached LLVM Source id: cache-llvm-source uses: actions/cache@v3 with: - path: mlir/llvm-project + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/llvm-project key: Linux-llvm-${{ needs.constants.outputs.llvm_version }}-generic-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -196,7 +229,7 @@ jobs: id: cache-llvm-build uses: actions/cache@v3 with: - path: llvm-build + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ needs.constants.outputs.llvm_version }}-${{matrix.python_version}}-generic-build fail-on-cache-miss: True @@ -204,7 +237,7 @@ jobs: id: cache-mhlo-source uses: actions/cache@v3 with: - path: mlir/mlir-hlo + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/mlir-hlo key: Linux-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-source enableCrossOsArchive: True fail-on-cache-miss: True @@ -213,7 +246,7 @@ jobs: id: cache-mhlo-build uses: actions/cache@v3 with: - path: mhlo-build + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build key: ${{ runner.os }}-${{ runner.arch }}-mhlo-${{ needs.constants.outputs.mhlo_version }}-generic-build fail-on-cache-miss: True @@ -221,7 +254,7 @@ jobs: id: cache-enzyme-source uses: actions/cache@v3 with: - path: mlir/Enzyme + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme key: Linux-enzyme-${{ needs.constants.outputs.enzyme_version }}-generic-source fail-on-cache-miss: True @@ -229,7 +262,7 @@ jobs: id: cache-enzyme-build uses: actions/cache@v3 with: - path: enzyme-build + path: ${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build key: ${{ runner.os }}-${{ runner.arch }}-enzyme-${{ needs.constants.outputs.llvm_version }}-${{ needs.constants.outputs.enzyme_version }}-generic-build fail-on-cache-miss: True @@ -244,16 +277,21 @@ jobs: -DENABLE_LIGHTNING_KOKKOS=ON \ -DLIGHTNING_GIT_TAG="v0.34.0" \ -DKokkos_ENABLE_SERIAL=ON \ - -DKokkos_ENABLE_OPENMP=OFF \ + -DKokkos_ENABLE_OPENMP=ON \ -DKokkos_ENABLE_COMPLEX_ALIGN=OFF \ + -DOpenMP_ROOT=$(brew --prefix libomp) \ -DENABLE_WARNINGS=OFF \ -DENABLE_OPENQASM=ON \ - -DENABLE_OPENMP=OFF + -DENABLE_OPENMP=OFF \ + -DLQ_ENABLE_KERNEL_OMP=OFF cmake --build runtime-build --target rt_capi rtd_lightning rtd_openqasm - name: Test Catalyst-Runtime + env: + VENV_SITE_PACKAGES: ${{ steps.setup_env.outputs.venv_site_packages }} run: | + export PYTHONPATH="$VENV_SITE_PACKAGES:$PYTHONPATH" python${{ matrix.python_version }} -m pip install amazon-braket-pennylane-plugin cmake --build runtime-build --target runner_tests_lightning runner_tests_openqasm ./runtime-build/tests/runner_tests_lightning @@ -268,32 +306,38 @@ jobs: -DQUANTUM_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ - -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ - -DMHLO_DIR=$GITHUB_WORKSPACE/mhlo-build/lib/cmake/mlir-hlo \ - -DMHLO_BINARY_DIR=$GITHUB_WORKSPACE/mhlo-build/bin \ - -DEnzyme_DIR=$GITHUB_WORKSPACE/enzyme-build \ - -DENZYME_SRC_DIR=$GITHUB_WORKSPACE/mlir/Enzyme \ + -DMLIR_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build/lib/cmake/mlir \ + -DMHLO_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build/lib/cmake/mlir-hlo \ + -DMHLO_BINARY_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build/bin \ + -DEnzyme_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build \ + -DENZYME_SRC_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/mlir/Enzyme \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DLLVM_ENABLE_LLD=OFF + -DLLVM_ENABLE_ZSTD=FORCE_ON \ + -DLLVM_ENABLE_LLD=OFF \ + -DLLVM_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build/lib/cmake/llvm cmake --build quantum-build --target check-dialects compiler_driver - name: Build wheel run: | PYTHON=python${{ matrix.python_version }} \ - LLVM_BUILD_DIR=$GITHUB_WORKSPACE/llvm-build \ - MHLO_BUILD_DIR=$GITHUB_WORKSPACE/mhlo-build \ + LLVM_BUILD_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/llvm-build \ + MHLO_BUILD_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/mhlo-build \ DIALECTS_BUILD_DIR=$GITHUB_WORKSPACE/quantum-build \ RT_BUILD_DIR=$GITHUB_WORKSPACE/runtime-build \ - ENZYME_BUILD_DIR=$GITHUB_WORKSPACE/enzyme-build \ + ENZYME_BUILD_DIR=${{ steps.setup_env.outputs.dependency_build_dir }}/enzyme-build \ make wheel + - name: Repair wheel using delocate-wheel + run: | + # ignore-missing-dependencies only ignores libopenblas.dylib + delocate-wheel --require-archs=arm64 -w ./wheel -v dist/*.whl --ignore-missing-dependencies -vv + - name: Upload Wheel Artifact uses: actions/upload-artifact@v3 with: name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip - path: dist/ + path: wheel/ retention-days: 14 test-wheels: @@ -301,28 +345,29 @@ jobs: strategy: fail-fast: false matrix: - os: [macos-latest-xlarge] - arch: [arm64] python_version: ${{ fromJson(needs.constants.outputs.python_versions) }} # To check all wheels for supported python3 versions - name: Test Wheels (Python ${{ matrix.python_version }}) on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Test Wheels (Python ${{ matrix.python_version }}) on Xanadu M2 Mac + runs-on: + group: 'Office 24th floor M2 Mac' steps: - name: Checkout Catalyst repo uses: actions/checkout@v3 + - name: Setup Runner Environment + id: setup_env + uses: ./.github/workflows/utils/setup_self_hosted_macos_env + with: + python_version: ${{ matrix.python_version }} + - name: Download Wheel Artifact uses: actions/download-artifact@v3 with: name: catalyst-macos_arm64-wheel-py-${{ matrix.python_version }}.zip path: dist - - name: Set up Python via brew - run: | - brew install python@${{ matrix.python_version }} - - name: Install Python dependencies run: | # tensorflow-cpu is not distributed for macOS ARM @@ -340,6 +385,6 @@ jobs: - name: Run Python Pytest Tests run: | - python${{ matrix.python_version }} -m pytest $GITHUB_WORKSPACE/frontend/test/pytest -n auto - python${{ matrix.python_version }} -m pytest $GITHUB_WORKSPACE/frontend/test/pytest --backend="lightning.kokkos" -n auto - python${{ matrix.python_version }} -m pytest $GITHUB_WORKSPACE/frontend/test/pytest --runbraket=LOCAL -n auto + python${{ matrix.python_version }} -m pytest -v $GITHUB_WORKSPACE/frontend/test/pytest -n auto + python${{ matrix.python_version }} -m pytest -v $GITHUB_WORKSPACE/frontend/test/pytest --backend="lightning.kokkos" -n auto + python${{ matrix.python_version }} -m pytest -v $GITHUB_WORKSPACE/frontend/test/pytest --runbraket=LOCAL -n auto diff --git a/.github/workflows/build-wheel-macos-x86_64.yaml b/.github/workflows/build-wheel-macos-x86_64.yaml index 7d29eaa1d8..dad7e3c586 100644 --- a/.github/workflows/build-wheel-macos-x86_64.yaml +++ b/.github/workflows/build-wheel-macos-x86_64.yaml @@ -125,11 +125,12 @@ jobs: -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_INSTALL_UTILS=ON \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_ZSTD=FORCE_ON \ -DLLVM_ENABLE_LLD=OFF \ -DMLIR_ENABLE_BINDINGS_PYTHON=ON \ -DPython3_EXECUTABLE=$(which python${{ matrix.python_version }}) \ - -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") + -DPython3_NumPy_INCLUDE_DIRS=$(python${{ matrix.python_version }} -c "import numpy as np; print(np.get_include())") \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden # TODO: when updating LLVM, test to see if mlir/unittests/Bytecode/BytecodeTest.cpp:55 is passing # and remove filter @@ -146,7 +147,8 @@ jobs: -DMLIR_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/mlir \ -DLLVM_ENABLE_LLD=OFF \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF + -DLLVM_ENABLE_ZSTD=FORCE_ON \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden cmake --build mhlo-build --target check-mlir-hlo @@ -156,7 +158,8 @@ jobs: cmake -S mlir/Enzyme/enzyme -B enzyme-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_DIR=$GITHUB_WORKSPACE/llvm-build/lib/cmake/llvm \ - -DENZYME_STATIC_LIB=ON + -DENZYME_STATIC_LIB=ON \ + -DCMAKE_CXX_VISIBILITY_PRESET=hidden cmake --build enzyme-build --target EnzymeStatic-18 @@ -183,7 +186,7 @@ jobs: - name: Install Dependencies (Python) run: | - python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML cmake ninja scipy + python${{ matrix.python_version }} -m pip install wheel numpy pybind11 PyYAML cmake ninja delocate - name: Get Cached LLVM Source id: cache-llvm-source @@ -238,6 +241,7 @@ jobs: # Build Catalyst-Runtime - name: Build Catalyst-Runtime run: | + # Segfaults in computing Lightning's adjoint-jacobian when building with OMP cmake -S runtime -B runtime-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=$GITHUB_WORKSPACE/runtime-build/lib \ @@ -251,7 +255,8 @@ jobs: -DKokkos_ENABLE_DEPRECATION_WARNINGS=OFF \ -DENABLE_WARNINGS=OFF \ -DENABLE_OPENQASM=ON \ - -DENABLE_OPENMP=OFF + -DENABLE_OPENMP=OFF \ + -DLQ_ENABLE_KERNEL_OMP=OFF cmake --build runtime-build --target rt_capi rtd_lightning rtd_openqasm @@ -277,7 +282,7 @@ jobs: -DEnzyme_DIR=$GITHUB_WORKSPACE/enzyme-build \ -DENZYME_SRC_DIR=$GITHUB_WORKSPACE/mlir/Enzyme \ -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_ZSTD=FORCE_ON \ -DLLVM_ENABLE_LLD=OFF cmake --build quantum-build --target check-dialects compiler_driver @@ -292,11 +297,16 @@ jobs: ENZYME_BUILD_DIR=$GITHUB_WORKSPACE/enzyme-build \ make wheel + - name: Repair wheel using delocate-wheel + run: | + # ignore-missing-dependencies only ignores libopenblas.dylib + delocate-wheel --require-archs=x86_64 -w ./wheel -v dist/*.whl --ignore-missing-dependencies -vv + - name: Upload Wheel Artifact uses: actions/upload-artifact@v3 with: name: catalyst-macos_x86_64-wheel-py-${{ matrix.python_version }}.zip - path: dist/ + path: wheel/ retention-days: 14 test-wheels: diff --git a/.github/workflows/utils/setup_self_hosted_macos_env/action.yml b/.github/workflows/utils/setup_self_hosted_macos_env/action.yml new file mode 100644 index 0000000000..786624b88a --- /dev/null +++ b/.github/workflows/utils/setup_self_hosted_macos_env/action.yml @@ -0,0 +1,122 @@ +name: Setup PATH for Self-Hosted MacOS Runner +description: | + This workflow sets up the workflow environment for the Self-Hosted MacOS runner. + It should be called right after actions/checkout. + + +inputs: + python_version: + description: Version of Python being used by the Job + required: true + +outputs: + python_version: + description: 'The version of Python that was setup' + value: ${{ steps.python_venv.outputs.python_version }} + python_path: + description: 'Full absolute path to the python binary' + value: ${{ steps.python_venv.outputs.python_path }} + venv_location: + description: 'The path to the virtualenv created for python' + value: ${{ steps.python_venv.outputs.venv_location }} + venv_site_packages: + description: 'The path to the site-packages folder for the python venv created' + value: ${{ steps.site_packages.outputs.site_packages_dir }} + dependency_build_dir: + description: 'The directory where the LLVM/MLIR/MHLO sources should reside in' + value: ${{ steps.build_dir.outputs.dir_name }} + + +runs: + using: composite + steps: + + # These environment variables ensure multi-threading workloads (ex: pytest-xdist) + # work without any issues. Without them, the tests take hours to complete. + - name: Setup MacOS Env Vars + shell: bash + run: | + # The following two environment variables are set to ensure no threading related commands are blocked/slowed + # by the OS. They may or may not be needed but added to ensure there are no slowdowns. + echo 'no_proxy=*' >> $GITHUB_ENV + echo 'OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES' >> $GITHUB_ENV + + # The following two flags are exported to ensure the correct homebrew installed libomp libraries are used + # during build time. + echo "LDFLAGS=-L/opt/homebrew/opt/libomp/lib $LDFLAGS" >> $GITHUB_ENV + echo "CPPFLAGS=-I/opt/homebrew/opt/libomp/include $CPPFLAGS" >> $GITHUB_ENV + + + # These binaries are added to PATH as there were issues adding them by default on job start + - name: Add additional binaries to PATH + shell: bash + run: | + additional_paths=("/opt/homebrew/bin" "$HOME/.pyenv/shims" "$HOME/.cargo/bin") + for _path in ${additional_paths[@]}; + do + [[ ":$PATH:" != *":$_path:"* ]] && echo "$_path" >> $GITHUB_PATH + done + echo "PYENV_ROOT=$HOME/.pyenv" >> $GITHUB_ENV + + - name: Setup Python ${{ inputs.python_version }} + id: python_setup + shell: bash + env: + input_python_version: ${{ inputs.python_version }} + run: | + # Check if the requested Python version exists on the runner + pyenv versions | grep "$input_python_version" + + if [ $? -ne 0 ]; then + echo "Installing Python $input_python_version" + pyenv install "$input_python_version" + fi + + - name: Setup Python venv + id: python_venv + shell: bash + env: + input_python_version: ${{ inputs.python_version }} + venv_name: py_venv_${{ inputs.python_version }}_${{ github.sha }} + run: | + rm -rf "$venv_name" + + echo "$input_python_version" > "$GITHUB_WORKSPACE/.python-version" + + python -m venv "$venv_name" + + echo "$GITHUB_WORKSPACE/$venv_name/bin" >> $GITHUB_PATH + echo "venv_location=$GITHUB_WORKSPACE/$venv_name" >> $GITHUB_OUTPUT + + PY_PATH="$GITHUB_WORKSPACE/$venv_name/bin/python" + PY_VERSION=$(python --version) + echo "python_path=$PY_PATH" >> $GITHUB_OUTPUT + echo "python_version=$PY_VERSION" >> $GITHUB_OUTPUT + + - name: Get site-packages path + id: site_packages + shell: bash + run: | + echo "site_packages_dir=$(python -c 'from distutils.sysconfig import get_python_lib; print(get_python_lib())')" >> $GITHUB_OUTPUT + + - name: Display Python Setup + shell: bash + run: | + set -x + which python + which pip + python --version + pip --version + set +x + + - name: Setup Build Dir + id: build_dir + shell: bash + env: + BUILD_DIR: /tmp/llvm_cache_${{ inputs.python_version }} + run: | + rm -rf "$BUILD_DIR"* + mkdir "$BUILD_DIR" + + echo "dir_name=${BUILD_DIR}" >> $GITHUB_OUTPUT + diff --git a/Makefile b/Makefile index 46f7adf042..dbf23d2e48 100644 --- a/Makefile +++ b/Makefile @@ -222,11 +222,11 @@ endif $(MAKE) -C mlir format $(MAKE) -C runtime format ifdef check - python3 ./bin/format.py --check $(if $(version:-=),--cfversion $(version)) ./frontend/catalyst/utils + $(PYTHON) ./bin/format.py --check $(if $(version:-=),--cfversion $(version)) ./frontend/catalyst/utils black --check --verbose . isort --check --diff . else - python3 ./bin/format.py $(if $(version:-=),--cfversion $(version)) ./frontend/catalyst/utils + $(PYTHON) ./bin/format.py $(if $(version:-=),--cfversion $(version)) ./frontend/catalyst/utils black . isort . endif diff --git a/doc/changelog.md b/doc/changelog.md index a09290de36..96fa68fea0 100644 --- a/doc/changelog.md +++ b/doc/changelog.md @@ -1,3 +1,65 @@ +# Release 0.4.1 + +