sync-huggingface-repos #58
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: sync-huggingface-repos | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| actions: read | |
| defaults: | |
| run: | |
| shell: bash | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| maximize-build-space: | |
| description: 'Maximize build space.' | |
| required: false | |
| type: boolean | |
| default: true | |
| maximize-build-space-in-deepth: | |
| description: 'Maximize build space in deepth.' | |
| required: false | |
| type: boolean | |
| default: false | |
| huggingface-repository: | |
| description: 'Source HuggingFace repository, inform of "owner/name".' | |
| required: true | |
| type: string | |
| include: | |
| description: 'Glob patterns to match files to sync.' | |
| required: false | |
| type: string | |
| exclude: | |
| description: 'Glob patterns to exclude from files to sync.' | |
| required: false | |
| type: string | |
| modelscope-repository: | |
| description: 'Target ModelScope repository, inform of "owner/name", usually the same as Source HuggingFace repository.' | |
| required: false | |
| type: string | |
| jobs: | |
| sync: | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Maximize Build Space | |
| if: ${{ inputs.maximize-build-space }} | |
| uses: gpustack/.github/.github/actions/maximize-build-space@main | |
| with: | |
| root-reserve-mb: 1024 | |
| temp-reserve-mb: 1024 | |
| swap-size-mb: 1024 | |
| deep-clean: ${{ inputs.maximize-build-space-in-deepth }} | |
| - name: Deps | |
| run: | | |
| #!/usr/bin/env bash | |
| pip install -U "huggingface_hub[cli]" | |
| pip install -U "modelscope" | |
| - name: Download | |
| run: | | |
| #!/usr/bin/env bash | |
| huggingface-cli login --token ${{ secrets.CI_HUGGINGFACE_TOKEN }} | |
| df -h | |
| ls -alth ${{ inputs.huggingface-repository }} || true | |
| huggingface-cli download ${{ inputs.huggingface-repository }} \ | |
| --local-dir ${{ inputs.huggingface-repository }} \ | |
| ${{ inputs.include && format('--include "{0}"', inputs.include) }} \ | |
| ${{ inputs.exclude && format('--exclude "{0}"', inputs.exclude) }} | |
| ls -alth ${{ inputs.huggingface-repository }} || true | |
| df -h | |
| - name: Upload | |
| run: | | |
| #!/usr/bin/env bash | |
| modelscope login --token ${{ secrets.CI_MODELSCOPE_TOKEN }} | |
| df -h | |
| ls -alth ${{ inputs.huggingface-repository }} || true | |
| cd ${{ inputs.huggingface-repository }} && modelscope upload ${{ inputs.modelscope-repository && inputs.modelscope-repository || inputs.huggingface-repository}} \ | |
| . . \ | |
| ${{ inputs.include && format('--include "{0}"', inputs.include) }} \ | |
| ${{ inputs.exclude && format('--exclude "{0}"', inputs.exclude) }} | |
| ls -alth ${{ inputs.huggingface-repository }} || true | |
| df -h |