Skip to content

Commit cb3addf

Browse files
thxCodethxCode
committed
ci: support sync hf
Signed-off-by: thxCode <[email protected]>
1 parent d258417 commit cb3addf

File tree

5 files changed

+222
-0
lines changed

5 files changed

+222
-0
lines changed

‎.github/actions/maximize-space/README.md

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Maximize Space
2+
3+
This action maximizes the space available to building. It inspired by [easimon/maximize-build-space](https://github.com/easimon/maximize-build-space).
4+
5+
## Usage
6+
7+
```yaml
8+
jobs:
9+
build:
10+
runs-on: ubuntu-latest
11+
steps:
12+
- name: Maximize Build Space
13+
uses: gpustack/.github/actions/maximize-build-space@main
14+
```
15+
16+
## Inputs
17+
18+
```yaml
19+
inputs:
20+
deep-clean:
21+
description: 'Perform a deep clean, removing unnecessary packages.'
22+
required: false
23+
default: 'false'
24+
```

‎.github/actions/maximize-space/action.yaml

+89
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
name: 'Maximize Space'
2+
description: 'Maximize the space available to building'
3+
4+
inputs:
5+
deep-clean:
6+
description: 'Perform a deep clean, removing unnecessary packages.'
7+
required: false
8+
default: 'false'
9+
10+
runs:
11+
using: "composite"
12+
steps:
13+
- name: Pre
14+
shell: bash
15+
run: |
16+
#!/usr/bin/env bash
17+
18+
echo "Memory and swap:"
19+
sudo free
20+
echo
21+
sudo swapon --show
22+
echo
23+
24+
echo "Available storage:"
25+
sudo df -h
26+
echo
27+
- name: Action
28+
shell: bash
29+
run: |
30+
#!/usr/bin/env bash
31+
32+
if [[ ${{ inputs.deep-clean }} == 'true' ]]; then
33+
echo "Purging unnecessary packages"
34+
sudo apt-get remove -y '^aspnetcore-.*' || true
35+
sudo apt-get remove -y '^dotnet-.*' --fix-missing || true
36+
sudo apt-get remove -y '^llvm-.*' --fix-missing || true
37+
sudo apt-get remove -y 'php.*' --fix-missing || true
38+
sudo apt-get remove -y '^mongodb-.*' --fix-missing || true
39+
sudo apt-get remove -y '^mysql-.*' --fix-missing || true
40+
sudo apt-get remove -y '^postgresql-.*' --fix-missing || true
41+
sudo apt-get remove -y --fix-missing \
42+
azure-cli \
43+
firefox \
44+
powershell \
45+
mono-devel \
46+
libgl1-mesa-dri \
47+
microsoft-edge-stable \
48+
google-chrome-stable \
49+
google-cloud-sdk \
50+
google-cloud-cli \
51+
apache2 \
52+
python3 python3-dev python3-pip python3-venv || true
53+
sudo apt-get autoremove -y || true
54+
sudo apt-get clean || true
55+
fi
56+
57+
echo "Purging unnecessary directories"
58+
sudo rm -rf /usr/local/.ghcup || true
59+
sudo rm -rf /usr/local/julia* || true
60+
sudo rm -rf /usr/local/graalvm || true
61+
sudo rm -rf /usr/local/lib/android || true
62+
sudo rm -rf /usr/local/lib/node_modules || true
63+
sudo rm -rf /usr/local/share/chromium || true
64+
sudo rm -rf /usr/local/share/powershell || true
65+
sudo rm -rf /usr/share/dotnet || true
66+
sudo rm -rf /usr/share/swift || true
67+
sudo rm -rf /opt/ghc || true
68+
sudo rm -rf /opt/pipx || true
69+
sudo rm -rf /opt/google/chrome || true
70+
sudo rm -rf /opt/microsoft/msedge || true
71+
sudo rm -rf /opt/microsoft/powershell || true
72+
sudo rm -rf "${AGENT_TOOLSDIRECTORY:-/opt/hostedtoolcache/CodeQL}" || true
73+
74+
echo "Purging unnecessary images"
75+
sudo docker image prune --all --force
76+
- name: Post
77+
shell: bash
78+
run: |
79+
#!/usr/bin/env bash
80+
81+
echo "Memory and swap:"
82+
sudo free
83+
echo
84+
sudo swapon --show
85+
echo
86+
87+
echo "Available storage:"
88+
sudo df -h
89+
echo

‎.github/actions/mirror-release-gitcode/action.yml

+9
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ inputs:
2323
description: 'Tag name of the specific release to mirror, take precedence over max releases.'
2424
required: false
2525
default: ''
26+
code-only:
27+
description: 'Only synchroize codes.'
28+
required: false
29+
default: false
2630
github-token:
2731
description: 'Github token to donwload releases from Github repository, usually inherit from the composition action.'
2832
required: false
@@ -48,6 +52,7 @@ runs:
4852
GITCODE_REPOSITORY: "${{ inputs.gitcode-repository && inputs.gitcode-repository || (inputs.github-repository && inputs.github-repository || github.repository) }}"
4953
MAX_RELEASES: "${{ inputs.max-releases }}"
5054
SPECIFIC_RELEASE_TAG: "${{ inputs.specific-release-tag }}"
55+
CODE_ONLY: "${{ inputs.code-only }}"
5156
GITHUB_TOKEN: "${{ inputs.github-token && inputs.github-token || github.token }}"
5257
GITHUB_REPOSITORY: "${{ inputs.github-repository && inputs.github-repository || github.repository }}"
5358
DRY_RUN: "${{ inputs.dry-run == 'true' }}"
@@ -179,6 +184,10 @@ runs:
179184
git push gitcode --all --force
180185
git push gitcode --tags --force
181186
fi
187+
188+
if [[ "${CODE_ONLY}" == "true" ]]; then
189+
exit 0
190+
fi
182191
183192
info "Releasing to GitCode repository..."
184193
# oauth login to get token

‎.github/actions/mirror-release-gitee/action.yml

+9
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,10 @@ inputs:
2020
description: 'Tag name of the specific release to mirror, take precedence over max releases.'
2121
required: false
2222
default: ''
23+
code-only:
24+
description: 'Only synchroize codes.'
25+
required: false
26+
default: false
2327
github-token:
2428
description: 'Github token to donwload releases from Github repository, usually inherit from the composition action.'
2529
required: false
@@ -44,6 +48,7 @@ runs:
4448
GITEE_REPOSITORY: "${{ inputs.gitee-repository && inputs.gitee-repository || (inputs.github-repository && inputs.github-repository || github.repository) }}"
4549
MAX_RELEASES: "${{ inputs.max-releases }}"
4650
SPECIFIC_RELEASE_TAG: "${{ inputs.specific-release-tag }}"
51+
CODE_ONLY: "${{ inputs.code-only }}"
4752
GITHUB_TOKEN: "${{ inputs.github-token && inputs.github-token || github.token }}"
4853
GITHUB_REPOSITORY: "${{ inputs.github-repository && inputs.github-repository || github.repository }}"
4954
DRY_RUN: "${{ inputs.dry-run == 'true' }}"
@@ -136,6 +141,10 @@ runs:
136141
git push gitee --all --force
137142
git push gitee --tags --force
138143
fi
144+
145+
if [[ "${CODE_ONLY}" == "true" ]]; then
146+
exit 0
147+
fi
139148
140149
info "Releasing to Gitee repository..."
141150
# create gitee release if not exists

‎.github/workflows/sync-huggingface-repos.yml

+91
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
name: sync-huggingface-repos
2+
3+
permissions:
4+
contents: read
5+
pull-requests: read
6+
actions: read
7+
8+
defaults:
9+
run:
10+
shell: bash
11+
12+
on:
13+
workflow_dispatch:
14+
inputs:
15+
maximize-space:
16+
description: 'Maximize space.'
17+
required: false
18+
type: boolean
19+
default: true
20+
maximize-space-in-depth:
21+
description: 'Maximize space in deepth.'
22+
required: false
23+
type: boolean
24+
default: false
25+
huggingface-repository:
26+
description: 'Source HuggingFace repository, inform of "owner/name".'
27+
required: true
28+
type: string
29+
include:
30+
description: 'Glob patterns to match files to sync.'
31+
required: false
32+
type: string
33+
exclude:
34+
description: 'Glob patterns to exclude from files to sync.'
35+
required: false
36+
type: string
37+
modelscope-repository:
38+
description: 'Target HuggingFace repository, inform of "owner/name", usually the same as Source HuggingFace repository.'
39+
required: false
40+
type: string
41+
42+
43+
jobs:
44+
sync:
45+
runs-on: ubuntu-22.04
46+
steps:
47+
- name: Maximize Build Space
48+
if: ${{ inputs.maximize-space }}
49+
uses: gpustack/.github/.github/actions/maximize-space@main
50+
with:
51+
deep-clean: ${{ inputs.maximize-space-in-depth }}
52+
- name: Deps
53+
run: |
54+
#!/usr/bin/env bash
55+
56+
pip install -U "huggingface_hub[cli]"
57+
pip install -U "modelscope"
58+
- name: Login HuggingFace
59+
run: |
60+
#!/usr/bin/env bash
61+
62+
huggingface-cli login --token ${{ secrets.CI_HUGGINGFACE_TOKEN }}
63+
- name: Login ModelScope
64+
run: |
65+
#!/usr/bin/env bash
66+
67+
modelscope login --token ${{ secrets.CI_MODELSCOPE_TOKEN }}
68+
- name: Download
69+
run: |
70+
#!/usr/bin/env bash
71+
72+
huggingface-cli download ${{ inputs.huggingface-repository }} \
73+
--local-dir ${{ inputs.huggingface-repository }} \
74+
${{ inputs.include && format('--include "{0}"', inputs.include) }} \
75+
${{ inputs.exclude && format('--exclude "{0}"', inputs.exclude) }}
76+
- name: Upload
77+
run: |
78+
#!/usr/bin/env bash
79+
80+
cat <<EOF> /tmp/upload.py
81+
from modelscope.hub.api import HubApi
82+
83+
api = HubApi()
84+
api.push_model(
85+
model_id="${{ inputs.modelscope-repository && inputs.modelscope-repository || inputs.huggingface-repository }}",
86+
model_dir="${{ github.workspace }}/${{ inputs.huggingface-repository }}"
87+
)
88+
EOF
89+
90+
python --version
91+
python /tmp/upload.py

0 commit comments

Comments
 (0)