Python 3.13 and simplifications (#39)

spack · Oct 17, 2024 · ddcb14a · ddcb14a
1 parent eeb903a
commit ddcb14a
Show file tree

Hide file tree

Showing 20 changed files with 307 additions and 464 deletions.
diff --git a/.github/workflows/clingo.yml b/.github/workflows/clingo.yml
@@ -15,84 +15,50 @@ concurrency:
   group: clingo-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  SPACK_COLOR: always
+  SPACK_BACKTRACE: please
+  PYTHONUNBUFFERED: 1
+
 jobs:
-  macos_clingo_x86_64:
-    runs-on: macos-latest
+  macos_clingo:
+    runs-on: ${{ matrix.runner[0] }}
     strategy:
       matrix:
-        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.6", "3.7", "3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        # runner + macOS deployment target
+        runner: [["macos-13", "10.13"], ["macos-14", "11"]]
+        exclude:
+        - python-version: "3.6"
+          runner: ["macos-14", "11"]
+        - python-version: "3.7"
+          runner: ["macos-14", "11"]
+    env:
+      MACOSX_DEPLOYMENT_TARGET: ${{ matrix.runner[1] }}
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          repository: spack/spack
+          path: spack
+          ref: c710a1597f3566ab850d0ee8c82e71af04a08f9e
+      # See https://github.com/actions/setup-python/issues/960
+      - name: Workaround for broken Python versions from setup-python
+        run: brew install gettext zlib
       - uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Prepare Spack
+      - name: Install clingo-bootstrap
         run: |
           pip install --upgrade pip
           brew install bison
-          export PATH="/usr/local/opt/bison/bin:$PATH"
-          git clone https://github.com/spack/spack.git spack-src
-          cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f
-          if [ ${{ matrix.python-version }} = "3.12" ] ; then
-            # distutils -> setuptools patch
-            wget -q -O - https://github.com/spack/spack/commit/1452e927719ddbbf07118e88f9fd8d18fabf0fae.patch | git apply -v             
-          fi
-          cd ..
-          . spack-src/share/spack/setup-env.sh
+          export PATH="$(brew --prefix bison):$PATH"
+          . spack/share/spack/setup-env.sh
           spack external find --not-buildable cmake bison
           spack config add "config:install_tree:padded_length:256"
-      - name: Install clingo
-        run: |
-          . spack-src/share/spack/setup-env.sh
-          spack python clingo/scripts/install_clingo.py
-          mkdir -p binary-mirror
-          spack buildcache push --unsigned --force $PWD/binary-mirror clingo-bootstrap
-      - uses: actions/upload-artifact@v3
-        with:
-          name: clingo_binary_mirror
-          path: binary-mirror
 
-  macos_clingo_aarch64:
-    runs-on: ["self-hosted", "macOS", "ARM64"]
-    strategy:
-      matrix:
-        python-version: ["3.6.15", "3.7.14", "3.8.14", "3.9.14", "3.10.7", "3.11.5", "3.12.0"]
-    steps:
-      - uses: actions/checkout@v4
-      - name: Prepare Spack
-        env:
-          PYENV_VERSION: ${{ matrix.python-version }}
-        run: |
-          export PATH=/Users/spack-bootstrap-runner/homebrew/bin:/Users/spack-bootstrap-runner/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-          export PATH="$(pyenv prefix)/bin:$PATH"
-          python --version
-          
-          brew install cmake bison
-          export PATH="/Users/spack-bootstrap-runner/homebrew/opt/bison/bin:$PATH"
-          
-          pyenv exec pip install --upgrade pip
-          
-          git clone https://github.com/spack/spack.git spack-src
-          cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f
-          if [ ${{ matrix.python-version }} = "3.12.0" ] ; then
-            # distutils -> setuptools patch
-            wget -q -O - https://github.com/spack/spack/commit/1452e927719ddbbf07118e88f9fd8d18fabf0fae.patch | git apply -v             
-          fi
-          cd ..
-          . spack-src/share/spack/setup-env.sh
-          spack external find --not-buildable cmake bison
-          spack config add "config:install_tree:padded_length:256"
-      - name: Install clingo
-        env:
-          PYENV_VERSION: ${{ matrix.python-version }}
-        run: |
-          export PATH=/Users/spack-bootstrap-runner/homebrew/bin:/Users/spack-bootstrap-runner/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-          export PATH="$(pyenv prefix)/bin:$PATH"
-          python --version
-          . spack-src/share/spack/setup-env.sh
           spack python clingo/scripts/install_clingo.py
-          mkdir -p binary-mirror
-          spack buildcache push --unsigned --force $PWD/binary-mirror clingo-bootstrap
+          spack buildcache push --unsigned ./binary-mirror clingo-bootstrap
       - uses: actions/upload-artifact@v3
         with:
           name: clingo_binary_mirror
@@ -150,17 +116,15 @@ jobs:
       SPACK_MANYLINUX2014_TAG: ${{ needs.manylinux2014.outputs.spack_manylinux_tag }}
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          . ./clingo/scripts/copy_mirror_manylinux2014.sh
+      - run: ./copy_mirror_manylinux2014.sh
       - uses: actions/upload-artifact@v3
         with:
           name: clingo_binary_mirror
-          path: |
-            binary-mirror
+          path: binary-mirror
 
   clingo_json:
     runs-on: ubuntu-latest
-    needs: [ upload-manylinux2014, macos_clingo_x86_64, macos_clingo_aarch64 ]
+    needs: [ upload-manylinux2014, macos_clingo ]
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
@@ -171,9 +135,8 @@ jobs:
           name: clingo_binary_mirror
       - run: |
           pip install ruamel.yaml
-          python clingo/scripts/clingo_json.py
+          python3 clingo/scripts/clingo_json.py
       - uses: actions/upload-artifact@v3
         with:
           name: clingo_manifest
-          path: |
-            clingo.json
+          path: clingo.json
diff --git a/.github/workflows/gnupg.yml b/.github/workflows/gnupg.yml
@@ -15,42 +15,48 @@ concurrency:
   group: gnupg-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  SPACK_COLOR: always
+  SPACK_BACKTRACE: please
+  PYTHONUNBUFFERED: 1
+
 jobs:
   macos_gnupg:
     runs-on: ${{ matrix.runner[0] }}
     strategy:
       # List of:
       # 1. Runners
       # 2. Target architectures
+      # 3. macOS deployment target
       matrix:
         runner: [
-          ["macos-latest", "x86_64"], 
-          [["self-hosted", "macOS", "ARM64"], "aarch64"]
+          ["macos-13", "x86_64", "10.13"],  # highsierra
+          ["macos-14", "aarch64", "11"]     # bigsur
         ]
+    env:
+      MACOSX_DEPLOYMENT_TARGET: ${{ matrix.runner[2] }}
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          if [[ ${{ matrix.runner[1] }} == "aarch64" ]] ; then
-              export PATH=/Users/spack-bootstrap-runner/homebrew/bin:/Users/spack-bootstrap-runner/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-          fi
+      - uses: actions/checkout@v4
+        with:
+          repository: spack/spack
+          path: spack
+          ref: c710a1597f3566ab850d0ee8c82e71af04a08f9e
+      - name: Install gnupg
+        run: |
           brew install gawk perl
-          git clone https://github.com/spack/spack.git spack-src
-          cd spack-src && git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && cd ..
 
           # Disables internationalization to avoid linking to
           # libintl on MacOS, since that will make the binary
           # non portable
-          cd spack-src && git apply ../gnupg/patches/gnupg_macos.patch && cd ..    
+          git -C spack apply "$PWD/gnupg/patches/gnupg_macos.patch"
 
-          . spack-src/share/spack/setup-env.sh
+          . spack/share/spack/setup-env.sh
           spack external find --not-buildable gawk perl
-          spack config add "config:concretizer:original"
           spack config add "config:install_tree:padded_length:256"
-          mkdir -p binary-mirror
 
-          # Inject flags to have compatibility with MacOSX version 10.13 or higher
-          spack install gnupg cflags="-mmacosx-version-min=10.13" target=${{ matrix.runner[1] }}
-          spack buildcache push --unsigned --force $PWD/binary-mirror gnupg
+          spack install gnupg target=${{ matrix.runner[1] }}
+          spack buildcache push --unsigned ./binary-mirror gnupg
       - uses: actions/upload-artifact@v3
         with:
           name: gnupg_binary_mirror
@@ -106,31 +112,31 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          . ./gnupg/scripts/copy_mirror_manylinux2014.sh
+      - run: ./copy_mirror_manylinux2014.sh
       - uses: actions/upload-artifact@v3
         with:
           name: gnupg_binary_mirror
-          path: |
-            binary-mirror
+          path: binary-mirror
 
   gnupg_json:
     runs-on: ubuntu-latest
     needs: [ upload-manylinux2014, macos_gnupg ]
     steps:
       - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
+        with:
+          repository: spack/spack
+          path: spack
       - uses: actions/setup-python@v4
         with:
           python-version: 3.11
       - uses: actions/download-artifact@v3
         with:
           name: gnupg_binary_mirror
       - run: |
-          git clone https://github.com/spack/spack.git spack-src
-          . spack-src/share/spack/setup-env.sh
+          . spack/share/spack/setup-env.sh
           spack python gnupg/scripts/gnupg_json.py
       - uses: actions/upload-artifact@v3
         with:
           name: gnupg_manifest
-          path: |
-            gnupg.json
+          path: gnupg.json
diff --git a/.github/workflows/patchelf.yml b/.github/workflows/patchelf.yml
@@ -15,6 +15,11 @@ concurrency:
   group: patchelf-${{ github.ref }}
   cancel-in-progress: true
 
+env:
+  SPACK_COLOR: always
+  SPACK_BACKTRACE: please
+  PYTHONUNBUFFERED: 1
+
 jobs:
 
   manylinux2014:
@@ -67,13 +72,11 @@ jobs:
 
     steps:
       - uses: actions/checkout@v4
-      - run: |
-          . ./patchelf/scripts/copy_mirror_manylinux2014.sh
+      - run: ./copy_mirror_manylinux2014.sh
       - uses: actions/upload-artifact@v3
         with:
           name: patchelf_binary_mirror
-          path: |
-            binary-mirror
+          path: binary-mirror
 
   patchelf_json:
     runs-on: ubuntu-latest
@@ -88,9 +91,8 @@ jobs:
           name: patchelf_binary_mirror
       - run: |
           pip install ruamel.yaml
-          python patchelf/scripts/patchelf_json.py
+          python3 patchelf/scripts/patchelf_json.py
       - uses: actions/upload-artifact@v3
         with:
           name: patchelf_manifest
-          path: |
-            patchelf.json
+          path: patchelf.json
diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ For completeness we report a summary of Spack requirements below:
 
 Name | Supported Versions | System Requirement | Requirement Reason
 --- | -------------------|---------------------|--------------------
-Python | 3.6-3.12 | Yes | Interpreter for Spack
+Python | 3.6-3.13 | Yes | Interpreter for Spack
 C/C++ compilers | - | Yes | Building software
 GNU make | - | Yes | Building software
 patch | - | Yes | Building software
@@ -53,11 +53,11 @@ by compatibility with the [manylinux](https://github.com/pypa/manylinux) project
 
 Platform | OS | Compiler Toolchain | Architecture | Python
 ---------|----|--------------------|--------------|-------
-`linux` | `centos7` | `GCC 10.2.1`| `x86_64` | 3.6-3.12
-`linux` | `centos7` | `GCC 10.2.1`| `aarch64` | 3.6-3.12
-`linux` | `centos7` | `GCC 10.2.1`| `ppc64le` | 3.6-3.12
-`darwin`| `MacOS 10.13` or later | `Apple Clang 14.0.0` | `x86_64` | 3.6-3.12
-`darwin`| `MacOS 10.13` or later | `Apple Clang 14.0.3` | `aarch64` | 3.6-3.12
+`linux` | `centos7` | `GCC 10.2.1`| `x86_64` | 3.6-3.13
+`linux` | `centos7` | `GCC 10.2.1`| `aarch64` | 3.6-3.13
+`linux` | `centos7` | `GCC 10.2.1`| `ppc64le` | 3.6-3.13
+`darwin`| `MacOS 10.13` or later | `Apple Clang 14.0.0` | `x86_64` | 3.6-3.13
+`darwin`| `MacOS 10.13` or later | `Apple Clang 14.0.3` | `aarch64` | 3.6-3.13
 
 ## Github Actions Workflows
 

diff --git a/clingo/Dockerfile.manylinux2014 b/clingo/Dockerfile.manylinux2014
@@ -1,37 +1,32 @@
-FROM ghcr.io/spack/manylinux2014:v2023-10-03
+FROM ghcr.io/spack/manylinux2014:v2024-10-16
 
-RUN cd /opt/_internal && tar xvf static-libs-for-embedding-only.tar.xz && cd -
+RUN tar -C /opt/_internal -xvf /opt/_internal/static-libs-for-embedding-only.tar.xz
 
-RUN adduser -m spack
-
-# Switch to a non-root user
-USER spack
-ENV SPACK_CMD="/opt/python/cp311-cp311/bin/python /home/spack/spack/bin/spack"
-WORKDIR /home/spack
+WORKDIR /root
+ENV PATH="/root/spack/bin/:$PATH" \
+    SPACK_BACKTRACE=please \
+    SPACK_COLOR=always \
+    SPACK_PYTHON=/opt/python/cp311-cp311/bin/python \
+    PYTHONUNBUFFERED=1
 
 # Clone the repo and install Spack
-RUN git clone https://www.github.com/spack/spack.git && \
-    cd spack && \
-    git checkout -b docker-reference 08da9a854a53542e1f71d30ed8d14d6b7d18119f && \
-    cd ..
+RUN git clone https://github.com/spack/spack.git && \
+    git -C spack checkout -b docker-reference c710a1597f3566ab850d0ee8c82e71af04a08f9e
 
 # Set externals, locate compilers
-RUN ${SPACK_CMD} external find -j 1 --not-buildable bison cmake
-RUN ${SPACK_CMD} compiler find
-RUN ${SPACK_CMD} config add "config:concretizer:original"
-RUN ${SPACK_CMD} config add "config:install_tree:padded_length:256"
-
-# Run a script to build all the versions of clingo we could
-COPY --chown=spack:spack clingo/scripts/bootstrap_clingo_manylinux2014.sh /home/spack/bootstrap_clingo2014.sh
-COPY --chown=spack:spack clingo/scripts/install_clingo.py /home/spack/install_clingo.py
-
-RUN ${HOME}/bootstrap_clingo2014.sh 36
-RUN ${HOME}/bootstrap_clingo2014.sh 37
-RUN ${HOME}/bootstrap_clingo2014.sh 38
-RUN ${HOME}/bootstrap_clingo2014.sh 39
-RUN ${HOME}/bootstrap_clingo2014.sh 310
-RUN ${HOME}/bootstrap_clingo2014.sh 311
-RUN ${HOME}/bootstrap_clingo2014.sh 312
-
-COPY --chown=spack:spack clingo/scripts/create_binary_mirror.sh /home/spack/create_binary_mirror.sh
-RUN ${HOME}/create_binary_mirror.sh
+RUN spack external find -j 1 --not-buildable bison cmake
+RUN spack compiler find
+RUN spack config add "config:install_tree:padded_length:256"
+
+COPY clingo/scripts/install_clingo.py /root/install_clingo.py
+
+RUN SPACK_PYTHON=/opt/python/cp313-cp313/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp312-cp312/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp311-cp311/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp310-cp310/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp39-cp39/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp38-cp38/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp37-cp37m/bin/python3 spack python install_clingo.py
+RUN SPACK_PYTHON=/opt/python/cp36-cp36m/bin/python3 spack python install_clingo.py
+
+RUN spack buildcache push --unsigned ./binary-mirror $(spack --color=never find --hashes clingo-bootstrap)
diff --git a/clingo/packages.yaml b/clingo/packages.yaml