Skip to content

sync-huggingface-repos #58

sync-huggingface-repos

sync-huggingface-repos #58

name: sync-huggingface-repos
permissions:
contents: read
pull-requests: read
actions: read
defaults:
run:
shell: bash
on:
workflow_dispatch:
inputs:
maximize-build-space:
description: 'Maximize build space.'
required: false
type: boolean
default: true
maximize-build-space-in-deepth:
description: 'Maximize build space in deepth.'
required: false
type: boolean
default: false
huggingface-repository:
description: 'Source HuggingFace repository, inform of "owner/name".'
required: true
type: string
include:
description: 'Glob patterns to match files to sync.'
required: false
type: string
exclude:
description: 'Glob patterns to exclude from files to sync.'
required: false
type: string
modelscope-repository:
description: 'Target ModelScope repository, inform of "owner/name", usually the same as Source HuggingFace repository.'
required: false
type: string
jobs:
sync:
runs-on: ubuntu-22.04
steps:
- name: Maximize Build Space
if: ${{ inputs.maximize-build-space }}
uses: gpustack/.github/.github/actions/maximize-build-space@main
with:
root-reserve-mb: 1024
temp-reserve-mb: 1024
swap-size-mb: 1024
deep-clean: ${{ inputs.maximize-build-space-in-deepth }}
- name: Deps
run: |
#!/usr/bin/env bash
pip install -U "huggingface_hub[cli]"
pip install -U "modelscope"
- name: Download
run: |
#!/usr/bin/env bash
huggingface-cli login --token ${{ secrets.CI_HUGGINGFACE_TOKEN }}
df -h
ls -alth ${{ inputs.huggingface-repository }} || true
huggingface-cli download ${{ inputs.huggingface-repository }} \
--local-dir ${{ inputs.huggingface-repository }} \
${{ inputs.include && format('--include "{0}"', inputs.include) }} \
${{ inputs.exclude && format('--exclude "{0}"', inputs.exclude) }}
ls -alth ${{ inputs.huggingface-repository }} || true
df -h
- name: Upload
run: |
#!/usr/bin/env bash
modelscope login --token ${{ secrets.CI_MODELSCOPE_TOKEN }}
df -h
ls -alth ${{ inputs.huggingface-repository }} || true
cd ${{ inputs.huggingface-repository }} && modelscope upload ${{ inputs.modelscope-repository && inputs.modelscope-repository || inputs.huggingface-repository}} \
. . \
${{ inputs.include && format('--include "{0}"', inputs.include) }} \
${{ inputs.exclude && format('--exclude "{0}"', inputs.exclude) }}
ls -alth ${{ inputs.huggingface-repository }} || true
df -h