fix images

.gitignore

@@ -143,4 +143,5 @@ tmp
 local_cache/
 
 # test file for examples are generated and should not be committed
-docs/examples/**/test*.py
+docs/examples/**/test*.py
+compiled_requirements.txt

Makefile

@@ -102,11 +102,14 @@ publish-library: build-library
 	poetry publish
 
 test-build-images: build-library
-	# TODO: enable when we can remove special duckdb setting for python 3.12
+	# NOTE: poetry export does not work with our many different deps, we install a subset and freeze
 	# poetry export -f requirements.txt --output _gen_requirements.txt --without-hashes --extras gcp --extras redshift
-	# grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
+	poetry install --no-interaction -E gcp -E redshift -E duckdb
+	poetry run pip freeze > _gen_requirements.txt
+	# filter out libs that need native compilation
+	grep `cat compiled_packages.txt` _gen_requirements.txt > compiled_requirements.txt
 	docker build -f deploy/dlt/Dockerfile.airflow --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
-	# docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
+	docker build -f deploy/dlt/Dockerfile --build-arg=COMMIT_SHA="$(shell git log -1 --pretty=%h)" --build-arg=IMAGE_VERSION="$(shell poetry version -s)" .
 
 preprocess-docs:
 	# run docs preprocessing to run a few checks and ensure examples can be parsed

compiled_packages.txt

@@ -1 +1 @@
-cffi\|idna\|simplejson\|pendulum\|grpcio\|google-crc32c
+pyarrow\|cffi\|idna\|simplejson\|pendulum\|grpcio\|google-crc32c

deploy/dlt/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.15
+FROM python:3.11.11-alpine
 
 # Metadata
 LABEL org.label-schema.vendor="dltHub" \
@@ -17,16 +17,28 @@ ADD compiled_requirements.txt .
 
 # install alpine deps
 RUN apk update &&\
-    apk add --no-cache python3 ca-certificates curl postgresql &&\
-    apk add --no-cache --virtual build-deps build-base automake autoconf libtool python3-dev postgresql-dev libffi-dev linux-headers gcc musl-dev &&\
-    ln -s /usr/bin/python3 /usr/bin/python &&\
+    apk add --no-cache ca-certificates curl postgresql git &&\
+    apk add --no-cache --virtual build-deps build-base automake autoconf libtool python3-dev postgresql-dev libffi-dev linux-headers gcc musl-dev cmake &&\
     curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && python3 get-pip.py &&\
     rm get-pip.py &&\
-    pip3 install --upgrade setuptools wheel &&\
-    rm -r /usr/lib/python*/ensurepip &&\
-    pip3 install -r compiled_requirements.txt &&\
+    pip install --upgrade setuptools wheel pip &&\
+    rm -r /usr/lib/python*/ensurepip
+
+# install arrow 17.0.0, usually we would need apache-arrow-dev=17.0.0 but it is not available in alpine 3.20
+# adapt this version to the arrow version you need
+RUN git clone --no-checkout https://github.com/apache/arrow.git /arrow \
+    && cd /arrow \
+    && git checkout tags/apache-arrow-17.0.0 \
+    && cd cpp \
+    && mkdir build \
+    && cd build \
+    && cmake -DARROW_CSV=ON -DARROW_JSON=ON -DARROW_FILESYSTEM=ON .. \
+    && make -j$(nproc) \
+    && make install
+
+RUN pip install -r compiled_requirements.txt &&\
     apk del --purge build-deps
-    #rm -r /root/.cache
+
 
 # add build labels and envs
 ARG COMMIT_SHA=""
@@ -38,7 +50,7 @@ ENV IMAGE_VERSION=${IMAGE_VERSION}
 
 # install exactly the same version of the library we used to build
 COPY dist/dlt-${IMAGE_VERSION}.tar.gz .
-RUN pip3 install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[gcp,redshift,duckdb]
+RUN pip install /tmp/pydlt/dlt-${IMAGE_VERSION}.tar.gz[gcp,redshift,duckdb]
 
 WORKDIR /
 RUN rm -r /tmp/pydlt

deploy/dlt/Dockerfile.airflow

@@ -1,4 +1,4 @@
-FROM apache/airflow:2.5.2-python3.8
+FROM apache/airflow:2.5.2-python3.9
 
 # Metadata
 LABEL org.label-schema.vendor="dltHub" \

pyproject.toml

@@ -69,8 +69,8 @@ psycopg2cffi = {version = ">=2.9.0", optional = true, markers="platform_python_i
 grpcio = {version = ">=1.50.0", optional = true}
 google-cloud-bigquery = {version = ">=2.26.0", optional = true}
 pyarrow = [
-    {version = ">=12.0.0,<18", markers = "python_version >= '3.9' and python_version < '3.13'"},
-    {version = ">=18.0.0", markers = "python_version >= '3.13'"}
+    {version = ">=12.0.0,<18", markers = "python_version >= '3.9' and python_version < '3.13'", optional = true},
+    {version = ">=18.0.0", markers = "python_version >= '3.13'", optional = true}
 ]
 duckdb = {version = ">=0.9", optional = true}
 # keep per-python version dependency as a reference