Skip to content

Commit c8be85f

Browse files
thxCodethxCode
committed
ci: support sync hf
Signed-off-by: thxCode <[email protected]>
1 parent d258417 commit c8be85f

File tree

5 files changed

+297
-0
lines changed

5 files changed

+297
-0
lines changed

‎.github/actions/maximize-build-space/README.md

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# Maximize Build Space
2+
3+
This action maximizes the space available to building. It inspired by [easimon/maximize-build-space](https://github.com/easimon/maximize-build-space).
4+
5+
## Usage
6+
7+
```yaml
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Maximize Build Space
13+
uses: gpustack/.github/actions/maximize-build-space@main
14+
```
15+
16+
## Inputs
17+
18+
```yaml
19+
inputs:
20+
root-reserve-mb:
21+
description: 'Space to be left free on the root filesystem, in Megabytes.'
22+
required: false
23+
default: '2048'
24+
temp-reserve-mb:
25+
description: 'Space to be left free on the temp filesystem (/mnt), in Megabytes.'
26+
required: false
27+
default: '2048'
28+
swap-size-mb:
29+
description: 'Swap space to create, in Megabytes.'
30+
required: false
31+
default: '2048'
32+
deep-clean:
33+
description: 'Perform a deep clean, removing unnecessary packages.'
34+
required: false
35+
default: 'false'
36+
```

‎.github/actions/maximize-build-space/action.yaml

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
name: 'Maximize Build Space'
2+
description: 'Maximize the space available to building'
3+
4+
inputs:
5+
root-reserve-mb:
6+
description: 'Space to be left free on the root filesystem, in Megabytes.'
7+
required: false
8+
default: '2048'
9+
temp-reserve-mb:
10+
description: 'Space to be left free on the temp filesystem (/mnt), in Megabytes.'
11+
required: false
12+
default: '2048'
13+
swap-size-mb:
14+
description: 'Swap space to create, in Megabytes.'
15+
required: false
16+
default: '2048'
17+
deep-clean:
18+
description: 'Perform a deep clean, removing unnecessary packages.'
19+
required: false
20+
default: 'false'
21+
22+
runs:
23+
using: "composite"
24+
steps:
25+
- name: Pre
26+
shell: bash
27+
run: |
28+
#!/usr/bin/env bash
29+
30+
echo "Memory and swap:"
31+
sudo free
32+
echo
33+
sudo swapon --show
34+
echo
35+
36+
echo "Available storage:"
37+
sudo df -h
38+
echo
39+
- name: Action
40+
shell: bash
41+
run: |
42+
#!/usr/bin/env bash
43+
44+
GITHUB_WORKSPACE_OWNER="$(stat -c '%U:%G' "${GITHUB_WORKSPACE}")"
45+
46+
if [[ ${{ inputs.deep-clean }} == 'true' ]]; then
47+
echo "Purging unnecessary packages"
48+
sudo apt-get remove -y '^aspnetcore-.*' || true
49+
sudo apt-get remove -y '^dotnet-.*' --fix-missing || true
50+
sudo apt-get remove -y '^llvm-.*' --fix-missing || true
51+
sudo apt-get remove -y 'php.*' --fix-missing || true
52+
sudo apt-get remove -y '^mongodb-.*' --fix-missing || true
53+
sudo apt-get remove -y '^mysql-.*' --fix-missing || true
54+
sudo apt-get remove -y '^postgresql-.*' --fix-missing || true
55+
sudo apt-get remove -y --fix-missing \
56+
azure-cli \
57+
firefox \
58+
powershell \
59+
mono-devel \
60+
libgl1-mesa-dri \
61+
microsoft-edge-stable \
62+
google-chrome-stable \
63+
google-cloud-sdk \
64+
google-cloud-cli \
65+
apache2 || true
66+
sudo apt-get autoremove -y || true
67+
sudo apt-get clean || true
68+
fi
69+
70+
echo "Purging unnecessary directories"
71+
sudo rm -rf /usr/local/.ghcup || true
72+
sudo rm -rf /usr/local/julia* || true
73+
sudo rm -rf /usr/local/graalvm || true
74+
sudo rm -rf /usr/local/lib/android || true
75+
sudo rm -rf /usr/local/lib/node_modules || true
76+
sudo rm -rf /usr/local/share/chromium || true
77+
sudo rm -rf /usr/local/share/powershell || true
78+
sudo rm -rf /usr/share/dotnet || true
79+
sudo rm -rf /usr/share/swift || true
80+
sudo rm -rf /opt/ghc || true
81+
sudo rm -rf /opt/google/chrome || true
82+
sudo rm -rf /opt/microsoft/msedge || true
83+
sudo rm -rf /opt/microsoft/powershell || true
84+
sudo rm -rf "${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache/CodeQL}" || true
85+
86+
echo "Purging unnecessary images"
87+
sudo docker image prune --all --force
88+
89+
# github runners have an active swap file in /mnt/swapfile
90+
# we want to reuse the temp disk, so first unmount swap and clean the temp disk
91+
echo "Removing swap"
92+
sudo swapoff -a
93+
sudo rm -f /mnt/swapfile
94+
95+
echo "Creating LVM PV on root fs"
96+
# create loop pv image on root fs
97+
ROOT_PV_LOOP_PATH="/pv.img"
98+
ROOT_RESERVE_KB=$(expr ${{ inputs.root-reserve-mb }} \* 1024)
99+
ROOT_FREE_KB=$(df --block-size=1024 --output=avail / | tail -1)
100+
ROOT_LVM_SIZE_KB=$(expr $ROOT_FREE_KB - $ROOT_RESERVE_KB)
101+
ROOT_LVM_SIZE_BYTES=$(expr $ROOT_LVM_SIZE_KB \* 1024)
102+
sudo touch "${ROOT_PV_LOOP_PATH}" && sudo fallocate -z -l "${ROOT_LVM_SIZE_BYTES}" "${ROOT_PV_LOOP_PATH}"
103+
export ROOT_LOOP_DEV=$(sudo losetup --find --show "${ROOT_PV_LOOP_PATH}")
104+
sudo pvcreate -f "${ROOT_LOOP_DEV}"
105+
106+
# create pv on temp disk
107+
echo "Creating LVM PV on temp fs"
108+
TMP_PV_LOOP_PATH="/mnt/pv.img"
109+
TMP_RESERVE_KB=$(expr ${{ inputs.temp-reserve-mb }} \* 1024)
110+
TMP_FREE_KB=$(df --block-size=1024 --output=avail /mnt | tail -1)
111+
TMP_LVM_SIZE_KB=$(expr $TMP_FREE_KB - $TMP_RESERVE_KB)
112+
TMP_LVM_SIZE_BYTES=$(expr $TMP_LVM_SIZE_KB \* 1024)
113+
sudo touch "${TMP_PV_LOOP_PATH}" && sudo fallocate -z -l "${TMP_LVM_SIZE_BYTES}" "${TMP_PV_LOOP_PATH}"
114+
export TMP_LOOP_DEV=$(sudo losetup --find --show "${TMP_PV_LOOP_PATH}")
115+
sudo pvcreate -f "${TMP_LOOP_DEV}"
116+
117+
VG_NAME=buildvg
118+
# create volume group from these pvs
119+
sudo vgcreate "${VG_NAME}" "${TMP_LOOP_DEV}" "${ROOT_LOOP_DEV}"
120+
121+
echo "Recreating swap"
122+
# create and activate swap
123+
sudo lvcreate -L "${{ inputs.swap-size-mb }}M" -n swap "${VG_NAME}"
124+
sudo mkswap "/dev/mapper/${VG_NAME}-swap"
125+
sudo swapon "/dev/mapper/${VG_NAME}-swap"
126+
127+
echo "Recreating github workspace"
128+
# create and mount build volume
129+
sudo lvcreate -l 100%FREE -n buildlv "${VG_NAME}"
130+
sudo mkfs.ext4 -Enodiscard -m0 "/dev/mapper/${VG_NAME}-buildlv"
131+
sudo mount "/dev/mapper/${VG_NAME}-buildlv" "${GITHUB_WORKSPACE}"
132+
sudo chown -R "${GITHUB_WORKSPACE_OWNER}" "${GITHUB_WORKSPACE}"
133+
134+
- name: Post
135+
shell: bash
136+
run: |
137+
#!/usr/bin/env bash
138+
139+
echo "Memory and swap:"
140+
sudo free
141+
echo
142+
sudo swapon --show
143+
echo
144+
145+
echo "Available storage:"
146+
sudo df -h
147+
echo

‎.github/actions/mirror-release-gitcode/action.yml

+9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ inputs:
2323
description: 'Tag name of the specific release to mirror, take precedence over max releases.'
2424
required: false
2525
default: ''
26+
code-only:
27+
description: 'Only synchroize codes.'
28+
required: false
29+
default: false
2630
github-token:
2731
description: 'Github token to donwload releases from Github repository, usually inherit from the composition action.'
2832
required: false
@@ -48,6 +52,7 @@ runs:
4852
GITCODE_REPOSITORY: "${{ inputs.gitcode-repository && inputs.gitcode-repository || (inputs.github-repository && inputs.github-repository || github.repository) }}"
4953
MAX_RELEASES: "${{ inputs.max-releases }}"
5054
SPECIFIC_RELEASE_TAG: "${{ inputs.specific-release-tag }}"
55+
CODE_ONLY: "${{ inputs.code-only }}"
5156
GITHUB_TOKEN: "${{ inputs.github-token && inputs.github-token || github.token }}"
5257
GITHUB_REPOSITORY: "${{ inputs.github-repository && inputs.github-repository || github.repository }}"
5358
DRY_RUN: "${{ inputs.dry-run == 'true' }}"
@@ -179,6 +184,10 @@ runs:
179184
git push gitcode --all --force
180185
git push gitcode --tags --force
181186
fi
187+
188+
if [[ "${CODE_ONLY}" == "true" ]]; then
189+
exit 0
190+
fi
182191
183192
info "Releasing to GitCode repository..."
184193
# oauth login to get token

‎.github/actions/mirror-release-gitee/action.yml

+9
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ inputs:
2020
description: 'Tag name of the specific release to mirror, take precedence over max releases.'
2121
required: false
2222
default: ''
23+
code-only:
24+
description: 'Only synchroize codes.'
25+
required: false
26+
default: false
2327
github-token:
2428
description: 'Github token to donwload releases from Github repository, usually inherit from the composition action.'
2529
required: false
@@ -44,6 +48,7 @@ runs:
4448
GITEE_REPOSITORY: "${{ inputs.gitee-repository && inputs.gitee-repository || (inputs.github-repository && inputs.github-repository || github.repository) }}"
4549
MAX_RELEASES: "${{ inputs.max-releases }}"
4650
SPECIFIC_RELEASE_TAG: "${{ inputs.specific-release-tag }}"
51+
CODE_ONLY: "${{ inputs.code-only }}"
4752
GITHUB_TOKEN: "${{ inputs.github-token && inputs.github-token || github.token }}"
4853
GITHUB_REPOSITORY: "${{ inputs.github-repository && inputs.github-repository || github.repository }}"
4954
DRY_RUN: "${{ inputs.dry-run == 'true' }}"
@@ -136,6 +141,10 @@ runs:
136141
git push gitee --all --force
137142
git push gitee --tags --force
138143
fi
144+
145+
if [[ "${CODE_ONLY}" == "true" ]]; then
146+
exit 0
147+
fi
139148
140149
info "Releasing to Gitee repository..."
141150
# create gitee release if not exists

‎.github/workflows/sync-huggingface-repos.yml

+96
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
name: sync-huggingface-repos
2+
3+
permissions:
4+
contents: read
5+
pull-requests: read
6+
actions: read
7+
8+
defaults:
9+
run:
10+
shell: bash
11+
12+
on:
13+
workflow_dispatch:
14+
inputs:
15+
maximize-build-space:
16+
description: 'Maximize build space.'
17+
required: false
18+
type: boolean
19+
default: true
20+
maximize-build-space-in-deepth:
21+
description: 'Maximize build space in deepth.'
22+
required: false
23+
type: boolean
24+
default: false
25+
huggingface-repository:
26+
description: 'Source HuggingFace repository, inform of "owner/name".'
27+
required: true
28+
type: string
29+
include:
30+
description: 'Glob patterns to match files to sync.'
31+
required: false
32+
type: string
33+
exclude:
34+
description: 'Glob patterns to exclude from files to sync.'
35+
required: false
36+
type: string
37+
modelscope-repository:
38+
description: 'Target HuggingFace repository, inform of "owner/name", usually the same as Source HuggingFace repository.'
39+
required: false
40+
type: string
41+
42+
43+
jobs:
44+
sync:
45+
runs-on: ubuntu-22.04
46+
steps:
47+
- name: Maximize Build Space
48+
if: ${{ inputs.maximize-build-space }}
49+
uses: gpustack/.github/.github/actions/maximize-build-space@main
50+
with:
51+
root-reserve-mb: 1024
52+
temp-reserve-mb: 1024
53+
swap-size-mb: 1024
54+
deep-clean: ${{ inputs.maximize-build-space-in-deepth }}
55+
- name: Deps
56+
run: |
57+
#!/usr/bin/env bash
58+
59+
pip install -U "huggingface_hub[cli]"
60+
pip install -U "modelscope"
61+
- name: Login HuggingFace
62+
run: |
63+
#!/usr/bin/env bash
64+
65+
huggingface-cli login --token ${{ secrets.CI_HUGGINGFACE_TOKEN }}
66+
- name: Login ModelScope
67+
run: |
68+
#!/usr/bin/env bash
69+
70+
modelscope login --token ${{ secrets.CI_MODELSCOPE_TOKEN }}
71+
- name: Download
72+
run: |
73+
#!/usr/bin/env bash
74+
75+
huggingface-cli download ${{ inputs.huggingface-repository }} \
76+
--local-dir ${{ inputs.huggingface-repository }} \
77+
${{ inputs.include && format('--include "{0}"', inputs.include) }} \
78+
${{ inputs.exclude && format('--exclude "{0}"', inputs.exclude) }}
79+
- name: Upload
80+
run: |
81+
#!/usr/bin/env bash
82+
83+
cat <<EOF> /tmp/upload.py
84+
from modelscope.hub.api import HubApi
85+
86+
api = HubApi()
87+
api.push_model(
88+
model_id="${{ inputs.modelscope-repository && inputs.modelscope-repository || inputs.huggingface-repository }}",
89+
model_dir="${{ github.workspace }}/${{ inputs.huggingface-repository }}"
90+
)
91+
EOF
92+
93+
df -h
94+
export TMPDIR=${{ github.workspace }}
95+
python /tmp/upload.py
96+
df -h

0 commit comments

Comments
 (0)