fix: compatibilty v2 new name (jina-ai#5784)

Signed-off-by: samsja <[email protected]>
Co-authored-by: Jina Dev Bot <[email protected]>

Author: samsja

.github/workflows/ci.yml

@@ -31,9 +31,9 @@ jobs:
         run: |
           pip install flake8
           # stop the build if there are Python syntax errors or undefined names
-          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/pb/jina_pb2.py,jina/proto/pb/jina_pb2_grpc.py,jina/proto/pb2/jina_pb2.py,jina/proto/pb2/jina_pb2_grpc.py
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/docarray_v1,jina/proto/docarray_v2
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/resources/,jina/proto/pb/jina_pb2.py,jina/proto/pb/jina_pb2_grpc.py,jina/proto/pb2/jina_pb2.py,jina/proto/pb2/jina_pb2_grpc.py
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/proto/docarray_v1,jina/proto/docarray_v2
 
   check-docstring:
     runs-on: ubuntu-latest
@@ -395,9 +395,9 @@ jobs:
       - name: Prepare environment
         run: |
           docker build -f Dockerfiles/test-pip.Dockerfile -t jinaai/jina:test-pip .
-          pip install -U "git+https://github.com/docarray/docarray@feat-rewrite-v2#egg=docarray[common]"
           python -m pip install --upgrade pip
           python -m pip install wheel
+          pip install -U "git+https://github.com/docarray/docarray@feat-rewrite-v2#egg=docarray[common]"
           pip install ".[common,devel,test]" --no-cache-dir
           pip install -U protobuf${{ matrix.protobuf-version }}
           jina

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     rev: 4.0.1
     hooks:
     -   id: flake8
-        exclude: ^(.git|__pycache__|docs/source/conf.py|old|build|dist|tests|jina/resources/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py)
+        exclude: ^(.git|__pycache__|docs/source/conf.py|old|build|dist|tests|jina/resources/|jina/proto/docarray_v1|jina/proto/docarray_v2)
         args:
           - --max-complexity=10
           - --max-line-length=127
@@ -13,7 +13,7 @@ repos:
   hooks:
   - id: darglint
     files: jina/
-    exclude: ^(jina/helloworld/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/)
+    exclude: ^(jina/helloworld/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/|jina/proto/docarray_v1|jina/proto/docarray_v2)
     args:
     - --message-template={path}:{line} {msg_id} {msg}
     - -s=sphinx
@@ -24,26 +24,28 @@ repos:
   hooks:
   -   id: pydocstyle
       files: jina/
-      exclude: ^(jina/helloworld/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/)
+      exclude: ^(jina/helloworld/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/|jina/proto/docarray_v1|jina/proto/docarray_v2)
       args:
       - --select=D101,D102,D103
 - repo: https://github.com/ambv/black
   rev: 22.3.0
   hooks:
   - id: black
     types: [python]
-    exclude: ^(jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/)
+    exclude: ^(jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/|jina/proto/docarray_v1|jina/proto/docarray_v2)
     args:
       - -S
 - repo: https://github.com/asottile/blacken-docs
   rev: v1.12.1
   hooks:
   -   id: blacken-docs
+      exclude: ^(jina/helloworld/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/|jina/proto/docarray_v1|jina/proto/docarray_v2)
       args:
         - -S
 - repo: https://github.com/pycqa/isort
   rev: 5.10.1
   hooks:
     - id: isort
+      exclude: ^(jina/helloworld/|jina/proto/pb/jina_pb2.py|jina/proto/pb/jina_pb2_grpc.py|jina/proto/pb2/jina_pb2.py|jina/proto/pb2/jina_pb2_grpc.py|docs/|jina/resources/|jina/proto/docarray_v1|jina/proto/docarray_v2)
       args: ["--profile", "black"]
 

jina/_docarray.py

@@ -1,6 +1,6 @@
 try:
     from docarray import BaseDoc as Document
-    from docarray import DocArray as DocumentArray
+    from docarray import DocList as DocumentArray
 
     docarray_v2 = True
 

jina/proto/README.md

@@ -0,0 +1,55 @@
+# Build proto
+
+Jina support two versions of protobuf, before 3.19 and after (which is a breaking change for python), therefore we have
+duplicate python file generation from proto based on the installed protobuf version.
+
+Moreover, jina is compatible with docarray v1 and docarray v2 that introduce breaking change in the proto definition.
+
+Therefore, we end with 2 proto files, one for each version of docarray that we support. (Note in the future 
+docarray v1 support will be dropped, and we will come back to have proto version)
+
+This complex setup lead to a not straightforward way to generate the python code from the proto
+
+this guide explain how to do it properly.
+
+# how to build the proto
+
+## 1. build docker image for protobuf generation
+
+
+``` cmd
+docker build -f Dockerfiles/protogen.Dockerfile -t jinaai/protogen:local .
+```
+
+This build the docker image that will be used to generate the python code from the proto for proto **before** 3.19
+
+``` cmd
+docker build -f Dockerfiles/protogen-3.21.Dockerfile -t jinaai/protogen-3.21:local .
+```
+
+This build the docker image that will be used to generate the python code from the proto for proto **after* 3.19
+
+## 2. generate the python code from the proto
+
+note: you need to be in the root of the repo to do the following steps
+
+### For DocArray v1
+
+``` cmd
+docker run -it -v $(pwd)/jina/proto/docarray_v1:/jina/proto jinaai/protogen:local
+```
+
+``` cmd
+docker run -it -v $(pwd)/jina/proto/docarray_v1:/jina/proto jinaai/protogen-3.21:local
+```
+
+### For DocArray v2
+
+``` cmd
+docker run -it -v $(pwd)/jina/proto/docarray_v2:/jina/proto jinaai/protogen:local
+```
+
+``` cmd
+docker run -it -v $(pwd)/jina/proto/docarray_v2:/jina/proto jinaai/protogen-3.21:local
+```
+

jina/proto/pb/__init__.py jina/proto/docarray_v1/__init__.py

@@ -8,7 +8,7 @@ set -e
 #
 # current dir: jina root (the one with README.md)
 # run the following in bash:
-# docker run -v $(pwd)/jina/proto:/jina/proto jinaai/protogen
+# docker run -v $(pwd)/jina/proto/docarray_v2:/jina/proto jinaai/protogen
 # finally, set back owner of the generated files using: sudo chown -R $(id -u ${USER}):$(id -g ${USER}) ./jina/proto
 
 # The protogen docker image can also be build locally using:
@@ -22,7 +22,7 @@ SRC_NAME="${MODULE}.proto"
 
 COMP_PROTO_OUT_NAME="${MODULE}_pb2.py"
 COMP_GRPC_OUT_NAME="${MODULE}_pb2_grpc.py"
-PB_NAME="${2:-pb2}"
+PB_NAME="${2:-pb}"
 OUT_FOLDER="${PB_NAME}/"
 
 VER_FILE=../__init__.py
@@ -50,7 +50,7 @@ printf "using linux sed syntax, if you are running this on mac, you may want to
 # sed -i '' -e 's/import\ jina_pb2\ as\ jina__pb2/from\ \.\ import\ jina_pb2\ as\ jina__pb2/' ${SRC_DIR}jina_pb2_grpc.py
 # for linux
 sed -i "s/import\ docarray_pb2/import\ docarray.proto.${PB_NAME}.docarray_pb2/" "${SRC_DIR}${OUT_FOLDER}jina_pb2.py"
-sed -i 's/import\ jina_pb2\ as\ jina__pb2/from\ \.\.\ import\ serializer\ as\ jina__pb2/' "${SRC_DIR}${OUT_FOLDER}jina_pb2_grpc.py"
+sed -i 's/import\ jina_pb2\ as\ jina__pb2/from\ \.\.\.\ import\ serializer\ as\ jina__pb2/' "${SRC_DIR}${OUT_FOLDER}jina_pb2_grpc.py"
 
 OLDVER=$(sed -n 's/^__proto_version__ = '\''\(.*\)'\''$/\1/p' $VER_FILE)
 printf "current proto version:\t\e[1;33m$OLDVER\e[0m\n"

jina/proto/build-proto.sh jina/proto/docarray_v1/build-proto.sh

@@ -3,7 +3,7 @@
 import grpc
 
 from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2
-from .. import serializer as jina__pb2
+from ... import serializer as jina__pb2
 
 
 class JinaDataRequestRPCStub(object):

jina/proto/docarray.proto jina/proto/docarray_v1/docarray.proto

@@ -1,9 +1,9 @@
 # Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
 """Client and server classes corresponding to protobuf-defined services."""
 import grpc
-
 from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2
-from .. import serializer as jina__pb2
+
+from ... import serializer as jina__pb2
 
 
 class JinaDataRequestRPCStub(object):

jina/proto/jina.proto jina/proto/docarray_v1/jina.proto

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+set -e
+
+# Do NOT use this directly, use jinaai/protogen image
+# use jinaai/protogen:v21 in order to use compiler version == 21 (creates pb/docarray_pb2.py)
+# and use jinaai/protogen:latest to use compiler version <= 20 (creates pb2/docarray_pb2.py)
+# make sure to use jinaai/protogen:v21 to avoid overwriting the module
+#
+# current dir: jina root (the one with README.md)
+# run the following in bash:
+# docker run -v $(pwd)/jina/proto/docarray_v2:/jina/proto jinaai/protogen
+# finally, set back owner of the generated files using: sudo chown -R $(id -u ${USER}):$(id -g ${USER}) ./jina/proto
+
+# The protogen docker image can also be build locally using:
+# docker build -f Dockerfiles/protogen.Dockerfile -t jinaai/protogen:local .
+# or
+# docker build -f Dockerfiles/protogen-3.21.Dockerfile -t jinaai/protogen-3.21:local .
+
+SRC_DIR=./
+MODULE=jina
+SRC_NAME="${MODULE}.proto"
+
+COMP_PROTO_OUT_NAME="${MODULE}_pb2.py"
+COMP_GRPC_OUT_NAME="${MODULE}_pb2_grpc.py"
+PB_NAME="${2:-pb}"
+OUT_FOLDER="${PB_NAME}/"
+
+VER_FILE=../__init__.py
+
+if [ "$#" -ne 1 ] && [ "$#" -ne 2 ]; then
+    echo "Error: Please specify the [PATH_TO_GRPC_PYTHON_PLUGIN], refer more details at " \
+      "https://docs.jina.ai/"
+    printf "\n"
+    echo "USAGE:"
+    printf "\t"
+    echo "bash ./build-proto.sh [PATH_TO_GRPC_PYTHON_PLUGIN]"
+    exit 1
+fi
+
+PLUGIN_PATH=${1}  # /Volumes/TOSHIBA-4T/Documents/grpc/bins/opt/grpc_python_plugin
+
+printf "\e[1;33mgenerating protobuf and grpc python interface\e[0m\n"
+
+protoc -I ${SRC_DIR} --python_out="${SRC_DIR}${OUT_FOLDER}" --grpc_python_out="${SRC_DIR}${OUT_FOLDER}" --plugin=protoc-gen-grpc_python=${PLUGIN_PATH} ${SRC_DIR}${SRC_NAME}
+
+printf "\e[1;33mfixing grpc import\e[0m\n"
+printf "using linux sed syntax, if you are running this on mac, you may want to comment out the sed for linux"
+# fix import bug in google generator
+# for mac
+# sed -i '' -e 's/import\ jina_pb2\ as\ jina__pb2/from\ \.\ import\ jina_pb2\ as\ jina__pb2/' ${SRC_DIR}jina_pb2_grpc.py
+# for linux
+sed -i "s/import\ docarray_pb2/import\ docarray.proto.${PB_NAME}.docarray_pb2/" "${SRC_DIR}${OUT_FOLDER}jina_pb2.py"
+sed -i 's/import\ jina_pb2\ as\ jina__pb2/from\ \.\.\.\ import\ serializer\ as\ jina__pb2/' "${SRC_DIR}${OUT_FOLDER}jina_pb2_grpc.py"
+
+OLDVER=$(sed -n 's/^__proto_version__ = '\''\(.*\)'\''$/\1/p' $VER_FILE)
+printf "current proto version:\t\e[1;33m$OLDVER\e[0m\n"
+
+NEWVER=$(echo $OLDVER | awk -F. -v OFS=. 'NF==1{print ++$NF}; NF>1{$NF=sprintf("%0*d", length($NF), ($NF+1)); print}')
+printf "bump proto version to:\t\e[1;32m$NEWVER\e[0m\n"
+
+# for mac
+# sed -i '' -e 's/^__proto_version__ = '\''\(.*\)'\''/__proto_version__ = '\'"$NEWVER"\''/' $VER_FILE
+# for linux
+sed -i 's/^__proto_version__ = '\''\(.*\)'\''/__proto_version__ = '\'"$NEWVER"\''/' $VER_FILE
+
+
+
+printf "\e[1;32mAll done!\e[0m\n"
+printf "if you are running this inside Docker container, you may manually bump proto version to:\t\e[1;32m$NEWVER\e[0m\n"

jina/proto/pb2/__init__.py jina/proto/docarray_v1/pb/__init__.py

@@ -0,0 +1,10 @@
+syntax = "proto3";
+
+package docarray;
+
+/*
+ this file is just a placeholder for the DA coming from jina._docarray dependency
+ */
+message DocListProto {
+
+}