Skip to content

Commit 71db6a0

Browse files
JoanFMalaeddine-13girishc13
authored
feat: add beta support to scale Stateful Executors with consensus using RAFT (jina-ai#5564)
Signed-off-by: Joan Fontanals Martinez <[email protected]> Co-authored-by: AlaeddineAbdessalem <[email protected]> Co-authored-by: Girish Chandrashekar <[email protected]>
1 parent 87fa2db commit 71db6a0

File tree

90 files changed

+10758
-558
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

90 files changed

+10758
-558
lines changed

‎.github/workflows/cd.yml

+226-19
Original file line numberDiff line numberDiff line change
@@ -70,16 +70,6 @@ jobs:
7070
git pull && git add . && git commit -m "update ${{env.JINA_VERSION}} due to ${{github.event_name}} on ${{github.repository}}" && git push
7171
fi
7272
73-
- name: Pre-release (.devN)
74-
run: |
75-
git fetch --depth=1 origin +refs/tags/*:refs/tags/*
76-
pip install twine wheel
77-
./scripts/release.sh
78-
env:
79-
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
80-
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
81-
JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }}
82-
8373
update-docker:
8474
needs: update-schema
8575
runs-on: ubuntu-latest
@@ -122,21 +112,45 @@ jobs:
122112
uses: actions/setup-python@v4
123113
with:
124114
python-version: ${{ matrix.python-version }}
115+
- name: Set up Golang
116+
uses: actions/setup-go@v2
117+
with:
118+
go-version: 1.19.5
119+
- name: Install dependencies
120+
run: |
121+
python -m pip install --upgrade pip
122+
pip install setuptools wheel
123+
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
124+
pip install cibuildwheel
125+
126+
- name: Build wheels with setuptools-golang-build-manylinux-wheel
127+
run: |
128+
setuptools-golang-build-manylinux-wheels --pythons cp37-cp37m
125129
- name: Prepare environment
126130
run: |
127131
docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
128132
python -m pip install --upgrade pip
129133
python -m pip install wheel
130-
pip install -U protobuf==3.20.3
131-
pip install ".[common,devel,test]" --no-cache-dir
132-
pip install -U "git+https://github.com/docarray/docarray@feat-rewrite-v2#egg=docarray[common]"
134+
WHEEL_FILE=$(ls dist/*whl)
135+
pip install "$WHEEL_FILE[common,devel,test]" --no-cache-dir
136+
pip install -U "docarray[common]"
137+
pip install -U protobuf${{ matrix.protobuf-version }}
133138
jina
134139
export JINA_LOG_LEVEL="ERROR"
135140
- name: Test
136141
id: test
137142
run: |
138143
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/docarray_v2
139-
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite/test_deployment_http_composite_docarray_v2.py
144+
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/deployment_http_composite
145+
echo "flag it as jina for codeoverage"
146+
echo "codecov_flag=jina" >> $GITHUB_OUTPUT
147+
timeout-minutes: 30
148+
env:
149+
JINA_AUTH_TOKEN: "${{ secrets.JINA_AUTH_TOKEN }}"
150+
- name: Test stateful
151+
id: test_stateful
152+
run: |
153+
JINA_LOG_LEVEL=DEBUG pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/integration/stateful
140154
echo "flag it as jina for codeoverage"
141155
echo "codecov_flag=jina" >> $GITHUB_OUTPUT
142156
timeout-minutes: 30
@@ -156,6 +170,185 @@ jobs:
156170
flags: ${{ steps.test.outputs.codecov_flag }}
157171
fail_ci_if_error: false
158172

173+
174+
mark-dev-N:
175+
runs-on: ubuntu-latest
176+
steps:
177+
- name: Check out repository
178+
uses: actions/checkout@v2
179+
180+
- name: Set up Python ${{ matrix.python }}
181+
uses: actions/setup-python@v2
182+
with:
183+
python-version: ${{ matrix.python }}
184+
185+
- id: computedevdelta
186+
run: |
187+
LAST_VER=$(git tag -l | sort -V | tail -n1)
188+
COMMITS_SINCE_LAST_VER=$(git rev-list $LAST_VER..HEAD --count)
189+
echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT
190+
INIT_FILE='jina/__init__.py'
191+
RELEASE_VER=$(sed -n '/^__version__/p' $INIT_FILE | cut -d \' -f2)
192+
echo "devdelta=$COMMITS_SINCE_LAST_VER" >> $GITHUB_OUTPUT
193+
echo "releasever=$RELEASE_VER" >> $GITHUB_OUTPUT
194+
outputs:
195+
devdelta: ${{ steps.computedevdelta.outputs.devdelta }}
196+
releasever: ${{ steps.computedevdelta.outputs.releasever }}
197+
198+
199+
# Build the wheels for Linux and macOS for Python 3.8 and newer
200+
build-wheels:
201+
runs-on: ${{ matrix.os }}
202+
needs: [mark-dev-N]
203+
strategy:
204+
# Ensure that a wheel builder finishes even if another fails
205+
fail-fast: false
206+
matrix:
207+
include:
208+
# linux
209+
- os: ubuntu-latest
210+
python: '3.7'
211+
python-manylinux-tag: "cp37-cp37m"
212+
- os: ubuntu-latest
213+
python: '3.8'
214+
python-manylinux-tag: "cp38-cp38"
215+
- os: ubuntu-latest
216+
python: '3.9'
217+
python-manylinux-tag: "cp39-cp39"
218+
- os: ubuntu-latest
219+
python: '3.10'
220+
python-manylinux-tag: "cp310-cp310"
221+
- os: ubuntu-latest
222+
python: '3.11'
223+
python-manylinux-tag: "cp311-cp311"
224+
225+
# MacOS emulated
226+
- os: macos-latest
227+
python: '3.7'
228+
python-cibuildwheels: '37'
229+
platform_id: macosx_x86_64
230+
- os: macos-latest
231+
python: '3.8'
232+
python-cibuildwheels: '38'
233+
platform_id: macosx_x86_64
234+
- os: macos-latest
235+
python: '3.9'
236+
python-cibuildwheels: '39'
237+
platform_id: macosx_x86_64
238+
- os: macos-latest
239+
python: '3.10'
240+
python-cibuildwheels: '310'
241+
platform_id: macosx_x86_64
242+
- os: macos-latest
243+
python: '3.11'
244+
python-cibuildwheels: '311'
245+
platform_id: macosx_x86_64
246+
247+
# MacOS native
248+
- os: macos-latest
249+
python: '3.8'
250+
python-cibuildwheels: '38'
251+
platform_id: macosx_arm64
252+
- os: macos-latest
253+
python: '3.9'
254+
python-cibuildwheels: '39'
255+
platform_id: macosx_arm64
256+
- os: macos-latest
257+
python: '3.10'
258+
python-cibuildwheels: '310'
259+
platform_id: macosx_arm64
260+
- os: macos-latest
261+
python: '3.11'
262+
python-cibuildwheels: '311'
263+
platform_id: macosx_arm64
264+
265+
steps:
266+
- name: Check out repository
267+
uses: actions/checkout@v2
268+
269+
- name: Set up Python ${{ matrix.python }}
270+
uses: actions/setup-python@v2
271+
with:
272+
python-version: ${{ matrix.python }}
273+
274+
- name: Set up Golang
275+
uses: actions/setup-go@v2
276+
with:
277+
go-version: 1.19.5
278+
279+
- name: Install dependencies
280+
run: |
281+
python -m pip install --upgrade pip
282+
pip install setuptools wheel
283+
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
284+
pip install cibuildwheel
285+
286+
- name: Update version
287+
run: |
288+
DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}}
289+
RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}}
290+
INIT_FILE='jina/__init__.py'
291+
DEV_VER=$RELEASE_VER".dev"$devdelta
292+
sed -i '/'"${$RELEASE_VER}"'/s/.*/'"${DEV_VER}"'/' "${INIT_FILE}"
293+
294+
- name: Build wheels with setuptools-golang-build-manylinux-wheel
295+
if: ${{ matrix.os == 'ubuntu-latest' }}
296+
run: |
297+
setuptools-golang-build-manylinux-wheels --pythons ${{ matrix.python-manylinux-tag }}
298+
299+
- name: Build wheels with cibuildwheels on macos
300+
env:
301+
CIBW_BUILD: cp${{ matrix.python-cibuildwheels }}-${{ matrix.platform_id }}
302+
CIBW_ARCHS: all
303+
CIBW_TEST_COMMAND: python -c "import jina; import jraft"
304+
CIBW_BUILD_VERBOSITY: 1
305+
if: ${{ matrix.os == 'macos-latest' }}
306+
run: |
307+
python -m cibuildwheel --output-dir dist
308+
309+
- name: Test wheels
310+
run: |
311+
WHEEL_FILE=$(ls dist/*.whl)
312+
python -m pip install $WHEEL_FILE
313+
python -c "import jraft"
314+
if: ${{ matrix.os != 'macos-latest' || matrix.platform_id != 'macosx_arm64' }} # runners do not necessarily have macos ARM, so cannot run this test for it
315+
316+
- name: Upload wheels as artifacts
317+
uses: actions/upload-artifact@v2
318+
with:
319+
path: dist/*.whl
320+
321+
pre-release:
322+
if: |
323+
!startsWith(github.event.head_commit.message, 'chore') &&
324+
!startsWith(github.event.head_commit.message, 'build: hotfix') &&
325+
!endsWith(github.event.head_commit.message, 'reformatted by jina-dev-bot')
326+
needs: [build-wheels, mark-dev-N]
327+
runs-on: ubuntu-latest
328+
steps:
329+
- uses: actions/[email protected]
330+
with:
331+
# submodules: true
332+
fetch-depth: 100
333+
- run: |
334+
truncate --size=24KB README.md > README-trunc.md
335+
336+
- name: Update version
337+
run: |
338+
DEVDELTA=${{needs.mark-dev-N.outputs.devdelta}}
339+
RELEASE_VER=${{needs.mark-dev-N.outputs.releasever}}
340+
INIT_FILE='jina/__init__.py'
341+
DEV_VER=$RELEASE_VER".dev"$devdelta
342+
sed -i '/'"${$RELEASE_VER}"'/s/.*/'"${DEV_VER}"'/' "${INIT_FILE}"
343+
- name: Pre-release (.devN)
344+
run: |
345+
pip install twine wheel
346+
./scripts/release.sh
347+
env:
348+
TWINE_USERNAME: ${{ secrets.TWINE_USERNAME }}
349+
TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
350+
JINA_SLACK_WEBHOOK: ${{ secrets.JINA_SLACK_WEBHOOK }}
351+
159352
core-test:
160353
needs: prep-testbed
161354
runs-on: ubuntu-latest
@@ -172,20 +365,34 @@ jobs:
172365
uses: actions/setup-python@v4
173366
with:
174367
python-version: ${{ matrix.python-version }}
368+
- name: Set up Golang
369+
uses: actions/setup-go@v2
370+
with:
371+
go-version: 1.19.5
372+
373+
- name: Install dependencies
374+
run: |
375+
python -m pip install --upgrade pip
376+
pip install setuptools wheel
377+
pip install git+https://github.com/jina-ai/setuptools-golang.git@feat-align-with-alaeddine-code
378+
pip install cibuildwheel
379+
380+
- name: Build wheels with setuptools-golang-build-manylinux-wheel
381+
run: |
382+
setuptools-golang-build-manylinux-wheels --pythons cp37-cp37m
175383
- name: Prepare environment
176384
run: |
177385
docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
178386
python -m pip install --upgrade pip
179387
python -m pip install wheel
180-
pip install ".[all]" --no-cache-dir
388+
WHEEL_FILE=$(ls dist/*whl)
389+
pip install "$WHEEL_FILE[all]" --no-cache-dir
181390
jina
182391
export JINA_LOG_LEVEL="ERROR"
183-
env:
184-
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
185392
- name: Test
186393
id: test
187394
run: |
188-
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/*' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }}
395+
pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml --timeout=600 -v -s --ignore-glob='tests/integration/docarray_v2/*' --ignore-glob='tests/integration/stateful/*' --ignore-glob='tests/integration/hub_usage/dummyhub*' ${{ matrix.test-path }}
189396
echo "flag it as jina for codeoverage"
190397
echo "codecov_flag=jina" >> $GITHUB_OUTPUT
191398
timeout-minutes: 30
@@ -476,7 +683,7 @@ jobs:
476683
# just for blocking the merge until all parallel core-test are successful
477684
success-all-steps:
478685
runs-on: ubuntu-latest
479-
needs: [core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-graceful-and-deployment-test, k8s-failures-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker]
686+
needs: [core-test, docarray-v-two-test, import-test, hub-test, k8s-flow-test, k8s-graceful-and-deployment-test, k8s-failures-test, docker-compose-test, docker-image-test, benchmark-pre-release, update-schema, update-docker, pre-release]
480687
if: always()
481688
steps:
482689
- uses: technote-space/workflow-conclusion-action@v2

0 commit comments

Comments
 (0)