diff --git a/.github/workflows/.lithops_config_aws.template b/.github/workflows/.lithops_config_aws.template new file mode 100644 index 00000000..a4c0470c --- /dev/null +++ b/.github/workflows/.lithops_config_aws.template @@ -0,0 +1,17 @@ +lithops: + backend: aws_lambda + storage: aws_s3 + storage_bucket: lithops-tom-data + log_level: WARN + +aws: + access_key_id: + secret_access_key: + +aws_lambda: + region_name: eu-west-1 + execution_role: arn:aws:iam::111560892610:role/lithops-execution-role + +aws_s3: + region_name: eu-west-1 + storage_bucket: lithops-tom-data diff --git a/.github/workflows/.lithops_config_gcp.template b/.github/workflows/.lithops_config_gcp.template new file mode 100644 index 00000000..6833e31a --- /dev/null +++ b/.github/workflows/.lithops_config_gcp.template @@ -0,0 +1,12 @@ +lithops: + backend: gcp_functions + storage: gcp_storage + log_level: WARN + +gcp: + region: europe-west2 + credentials_path: + +gcp_storage: + region: europe-west2 + storage_bucket: tom-lithops-data diff --git a/.github/workflows/lithops-images.yml b/.github/workflows/lithops-images.yml new file mode 100644 index 00000000..18288c0b --- /dev/null +++ b/.github/workflows/lithops-images.yml @@ -0,0 +1,86 @@ +name: Lithops images + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: ${{ matrix.lithops-backend }} ${{ matrix.os }} py${{ matrix.python_version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + python-version: "3.11" + requirements-file: ci/requirements-lithops-aws.txt + lithops-config: .github/workflows/.lithops_config_aws + lithops-backend: aws_lambda + lithops-build-file: ci/docker/Dockerfile_aws_lambda_main + lithops-memory: 2000 + - os: ubuntu-latest + python-version: "3.11" + requirements-file: ci/requirements-lithops-gcp.txt + lithops-config: .github/workflows/.lithops_config_gcp + lithops-backend: gcp_functions + lithops-build-file: ci/requirements-lithops-gcp.txt + lithops-memory: 2048 + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Copy lithops configuration templates + run: | + cp $GITHUB_WORKSPACE/.github/workflows/.lithops_config_aws.template $GITHUB_WORKSPACE/.github/workflows/.lithops_config_aws + cp $GITHUB_WORKSPACE/.github/workflows/.lithops_config_gcp.template $GITHUB_WORKSPACE/.github/workflows/.lithops_config_gcp + + - name: Google auth + id: 'auth' + uses: 'google-github-actions/auth@v2' + with: + credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}' + create_credentials_file: true + + - name: Configure lithops AWS + uses: microsoft/variable-substitution@v1 + with: + files: ${{ github.workspace }}/.github/workflows/.lithops_config_aws + env: + aws.access_key_id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws.secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + + - name: Configure lithops GCP + uses: microsoft/variable-substitution@v1 + with: + files: ${{ github.workspace }}/.github/workflows/.lithops_config_gcp + env: + gcp.credentials_path: ${{ steps.auth.outputs.credentials_file_path }} + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v3 + with: + python-version: ${{ matrix.python-version }} + architecture: x64 + + - name: Install + run: | + python -m pip install --upgrade pip + python -m pip install -r ${{ matrix.requirements-file }} + + - name: Build and deploy images + env: + LITHOPS_CONFIG_FILE: ${{ github.workspace }}/${{ matrix.lithops-config }} + run: | + lithops runtime build -b ${{ matrix.lithops-backend }} -f ${{ matrix.lithops-build-file }} cubed-runtime-main + lithops runtime deploy -b ${{ matrix.lithops-backend }} --memory ${{ matrix.lithops-memory }} cubed-runtime-main diff --git a/ci/docker/Dockerfile_aws_lambda_main b/ci/docker/Dockerfile_aws_lambda_main new file mode 100644 index 00000000..4524450c --- /dev/null +++ b/ci/docker/Dockerfile_aws_lambda_main @@ -0,0 +1,76 @@ +# Python 3.6 +#FROM python:3.6-slim-buster + +# Python 3.7 +#FROM python:3.7-slim-buster + +# Python 3.8 +#FROM python:3.8-slim-buster + +# Python 3.9 +#FROM python:3.9-slim-buster + +# Python 3.10 +#FROM python:3.10-slim-buster + +# Python 3.11 +FROM python:3.11-slim-buster + + +RUN apt-get update \ + # Install aws-lambda-cpp build dependencies + && apt-get install -y \ + g++ \ + make \ + cmake \ + unzip \ + git \ + # cleanup package lists, they are not used anymore in this image + && rm -rf /var/lib/apt/lists/* \ + && apt-cache search linux-headers-generic + +ARG FUNCTION_DIR="/function" + +# Copy function code +RUN mkdir -p ${FUNCTION_DIR} + +# Update pip +# NB botocore/boto3 are pinned due to https://github.com/boto/boto3/issues/3648 +# using versions from https://github.com/aio-libs/aiobotocore/blob/72b8dd5d7d4ef2f1a49a0ae0c37b47e5280e2070/setup.py +# due to s3fs dependency +RUN pip install --upgrade --ignore-installed pip wheel six setuptools \ + && pip install --upgrade --no-cache-dir --ignore-installed \ + awslambdaric \ + botocore==1.29.76 \ + boto3==1.26.76 \ + redis \ + httplib2 \ + requests \ + numpy \ + scipy \ + pandas \ + pika \ + kafka-python \ + cloudpickle \ + ps-mem \ + tblib + +# Set working directory to function root directory +WORKDIR ${FUNCTION_DIR} + +# Add Lithops +COPY lithops_lambda.zip ${FUNCTION_DIR} +RUN unzip lithops_lambda.zip \ + && rm lithops_lambda.zip \ + && mkdir handler \ + && touch handler/__init__.py \ + && mv entry_point.py handler/ + +# Put your dependencies here, using RUN pip install... or RUN apt install... + +RUN pip install \ + -e 'git+https://github.com/cubed-dev/cubed.git#egg=cubed' \ + s3fs + +ENTRYPOINT [ "/usr/local/bin/python", "-m", "awslambdaric" ] +CMD [ "handler.entry_point.lambda_handler" ] diff --git a/ci/requirements-lithops-aws.txt b/ci/requirements-lithops-aws.txt new file mode 100644 index 00000000..cc6babb1 --- /dev/null +++ b/ci/requirements-lithops-aws.txt @@ -0,0 +1,4 @@ +git+https://github.com/cubed-dev/cubed.git#egg=cubed +lithops[aws] +s3fs +rich diff --git a/ci/requirements-lithops-gcp.txt b/ci/requirements-lithops-gcp.txt new file mode 100644 index 00000000..d5b776c5 --- /dev/null +++ b/ci/requirements-lithops-gcp.txt @@ -0,0 +1,4 @@ +git+https://github.com/cubed-dev/cubed.git#egg=cubed +lithops[gcp] +gcsfs +rich