From 294ac5b0a3eb44b1b590cd5fa09dc7346b614587 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:02:46 +0300 Subject: [PATCH 01/16] Create .env Resolve warning messages "The POSTGRES_DB/USER variable is not set. Defaulting to a blank string." --- .env | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 000000000..6985d40d4 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +POSTGRES_USER=mediacms +POSTGRES_PASSWORD=mediacms +POSTGRES_DB=mediacms From 674ec35b03c81879f6a56d977375a7baea760b39 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:09:21 +0300 Subject: [PATCH 02/16] Update Dockerfile Update images and some packages; mitigate some CVEs. --- Dockerfile | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index af6ae1cd2..784723534 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.4-bookworm AS compile-image +FROM python:3.12.4-bookworm AS compile-image SHELL ["/bin/bash", "-c"] @@ -8,10 +8,11 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" ENV PIP_NO_CACHE_DIR=1 RUN mkdir -p /home/mediacms.io/mediacms/{logs} && cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV +RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-recommends # Install dependencies: COPY requirements.txt . - +RUN pip install pip==24.1.1 RUN pip install -r requirements.txt COPY . /home/mediacms.io/mediacms @@ -25,7 +26,7 @@ RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-637.x86_64-u rm Bento4-SDK-1-6-0-637.x86_64-unknown-linux.zip ############ RUNTIME IMAGE ############ -FROM python:3.11.4-bookworm as runtime-image +FROM python:3.12.4-bookworm as runtime-image ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 @@ -60,11 +61,7 @@ RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-stati rm -rf ffmpeg-tmp ffmpeg-release-amd64-static.tar.xz WORKDIR /home/mediacms.io/mediacms - EXPOSE 9000 80 - RUN chmod +x ./deploy/docker/entrypoint.sh - ENTRYPOINT ["./deploy/docker/entrypoint.sh"] - CMD ["./deploy/docker/start.sh"] From 9b8dbc905f8cca899e41cb3e792d65002544697a Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:11:36 +0300 Subject: [PATCH 03/16] Update requirements.txt Update some packages; Remove/mitigate some CVEs. --- requirements.txt | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/requirements.txt b/requirements.txt index acbb677a8..b0c9c5a23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,22 @@ -Django==4.2.2 -djangorestframework==3.14.0 -django-allauth==0.54.0 +setuptools==70.2.0 +Django==5.0.6 +djangorestframework==3.15.2 +django-allauth==0.55.2 psycopg==3.1.9 -uwsgi==2.0.21 -django-redis==5.3.0 +uwsgi==2.0.26 +django-redis==5.4.0 celery==5.3.1 -drf-yasg==1.21.6 -Pillow==9.5.0 -django-imagekit==4.1.0 -markdown==3.4.3 -django-filter==23.2 +drf-yasg==1.21.7 +Pillow==10.4.0 +django-imagekit==5.0.0 +markdown==3.6.0 +django-filter==24.2 filetype==1.2.0 -django-mptt==0.14.0 -django-crispy-forms==1.13.0 -requests==2.31.0 +django-mptt==0.16.0 +django-crispy-forms==2.2.0 +requests==2.32.3 django-celery-email==3.0.0 -m3u8==3.5.0 +m3u8==5.1.0 django-ckeditor==6.6.1 -django-debug-toolbar==4.1.0 +django-debug-toolbar==4.4.2 django-login-required-middleware==0.9.0 From b5070aabb1af085e312d553c66fc2e75d677656b Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:12:50 +0300 Subject: [PATCH 04/16] Update docker-compose.yaml Update postgres image. --- docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 4e4a3f564..77c5fc901 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -62,7 +62,7 @@ services: depends_on: - migrations db: - image: postgres:15.2-alpine + image: postgres:16.3-alpine volumes: - ../postgres_data:/var/lib/postgresql/data/ restart: always From 387e922014351e0b67f9881eebd47bc97f0a3d14 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:13:34 +0300 Subject: [PATCH 05/16] Update docker-compose-dev.yaml Update postgres image. --- docker-compose-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index 959dd6747..582d3736d 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -47,7 +47,7 @@ services: depends_on: - migrations db: - image: postgres:15.2-alpine + image: postgres:16.3-alpine volumes: - ../postgres_data:/var/lib/postgresql/data/ restart: always From 2dea733e0ad86dd97e39f4963e639dd9927ff379 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:13:58 +0300 Subject: [PATCH 06/16] Update docker-compose-http-proxy.yaml Update postgres image. --- docker-compose-http-proxy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-http-proxy.yaml b/docker-compose-http-proxy.yaml index 2b3194e26..229f790df 100644 --- a/docker-compose-http-proxy.yaml +++ b/docker-compose-http-proxy.yaml @@ -68,7 +68,7 @@ services: depends_on: - migrations db: - image: postgres:15.2-alpine + image: postgres:16.3-alpine volumes: - ../postgres_data/:/var/lib/postgresql/data/ restart: always @@ -89,4 +89,4 @@ services: test: ["CMD", "redis-cli","ping"] interval: 30s timeout: 10s - retries: 3 \ No newline at end of file + retries: 3 From 142ae75acd95bea9def92e3a64c296e6f0df4a77 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:14:17 +0300 Subject: [PATCH 07/16] Update docker-compose-https-proxy.yaml Update postgres image. --- docker-compose-https-proxy.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-https-proxy.yaml b/docker-compose-https-proxy.yaml index 51c4a7ab3..6d401837e 100644 --- a/docker-compose-https-proxy.yaml +++ b/docker-compose-https-proxy.yaml @@ -70,7 +70,7 @@ services: depends_on: - migrations db: - image: postgres:15.2-alpine + image: postgres:16.3-alpine volumes: - ../postgres_data/:/var/lib/postgresql/data/ restart: always @@ -91,4 +91,4 @@ services: test: ["CMD", "redis-cli","ping"] interval: 30s timeout: 10s - retries: 3 \ No newline at end of file + retries: 3 From 1c64938d21de64b9c06f0046b5b61652de17eb30 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:14:45 +0300 Subject: [PATCH 08/16] Update docker-compose-letsencrypt.yaml Update postgres image. --- docker-compose-letsencrypt.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-letsencrypt.yaml b/docker-compose-letsencrypt.yaml index b00db9d49..f1d20d49a 100644 --- a/docker-compose-letsencrypt.yaml +++ b/docker-compose-letsencrypt.yaml @@ -90,7 +90,7 @@ services: depends_on: - migrations db: - image: postgres:15.2-alpine + image: postgres:16.3-alpine volumes: - ../postgres_data:/var/lib/postgresql/data/ restart: always From 11f37ce2afc0247877e823be4c77a41e9697d9fb Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:15:05 +0300 Subject: [PATCH 09/16] Update docker-compose-named-volumes.yaml Update postgres image. --- docker-compose-named-volumes.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose-named-volumes.yaml b/docker-compose-named-volumes.yaml index 77058c325..e42c478c1 100644 --- a/docker-compose-named-volumes.yaml +++ b/docker-compose-named-volumes.yaml @@ -66,7 +66,7 @@ services: depends_on: - migrations db: - image: postgres:15.2-alpine + image: postgres:16.3-alpine volumes: - postgres_data:/var/lib/postgresql/data/ restart: always @@ -91,4 +91,4 @@ services: volumes: postgres_data: media_store: - static_store: \ No newline at end of file + static_store: From 5d1adc1d195abc49e785b1cc35192296e320e3d5 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:17:28 +0300 Subject: [PATCH 10/16] Update local_settings.py Set UPLOAD_MAX_SIZE = 100G --- deploy/docker/local_settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/docker/local_settings.py b/deploy/docker/local_settings.py index 7dbad9ad1..f4e79629a 100644 --- a/deploy/docker/local_settings.py +++ b/deploy/docker/local_settings.py @@ -4,6 +4,9 @@ POSTGRES_HOST = 'db' REDIS_LOCATION = "redis://redis:6379/1" +#Set max upload file size to 100G +UPLOAD_MAX_SIZE = 100000000000 + DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", From fbd252d0ab2a3ed1129e4d50337517942a526334 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:19:35 +0300 Subject: [PATCH 11/16] Update settings.py Set UPLOAD_MAX_SIZE = 100G --- cms/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cms/settings.py b/cms/settings.py index 33fd79671..0228a409c 100644 --- a/cms/settings.py +++ b/cms/settings.py @@ -83,8 +83,8 @@ # basic http auth user/password if you want to restrict access MAX_MEDIA_PER_PLAYLIST = 70 -# bytes, size of uploaded media -UPLOAD_MAX_SIZE = 800 * 1024 * 1000 * 5 +# bytes, size of uploaded media (100G) +UPLOAD_MAX_SIZE = 800 * 1024 * 1000 * 5 * 25 MAX_CHARS_FOR_COMMENT = 10000 # so that it doesn't end up huge TIMESTAMP_IN_TIMEBAR = False # shows timestamped comments in the timebar for videos From 11ab88b61f078c65e0a74104dfbcce0932fe9714 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:21:47 +0300 Subject: [PATCH 12/16] Update nginx.conf Unlimited file size for nginx. --- deploy/docker/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/nginx.conf b/deploy/docker/nginx.conf index 1dda6106c..a43254c71 100644 --- a/deploy/docker/nginx.conf +++ b/deploy/docker/nginx.conf @@ -10,7 +10,7 @@ events { http { proxy_connect_timeout 75; proxy_read_timeout 12000; - client_max_body_size 5800M; + client_max_body_size 0; sendfile on; tcp_nopush on; tcp_nodelay on; From ddafcc1c2230455046745046e5f4cbb252eb10fc Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Wed, 3 Jul 2024 07:23:07 +0300 Subject: [PATCH 13/16] Update nginx.conf --- deploy/local_install/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/local_install/nginx.conf b/deploy/local_install/nginx.conf index c13f67723..fc3c49f35 100644 --- a/deploy/local_install/nginx.conf +++ b/deploy/local_install/nginx.conf @@ -10,7 +10,7 @@ events { http { proxy_connect_timeout 75; proxy_read_timeout 12000; - client_max_body_size 5800M; + client_max_body_size 0; sendfile on; tcp_nopush on; tcp_nodelay on; From 43fe330e96239087dbd4704b77f51c91ab15cab1 Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Sat, 6 Jul 2024 23:53:00 +0300 Subject: [PATCH 14/16] Fix CVE-2024-39689 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 784723534..5559cb274 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update -y && apt-get -y upgrade && apt-get install --no-install-reco # Install dependencies: COPY requirements.txt . RUN pip install pip==24.1.1 +RUN pip install certifi RUN pip install -r requirements.txt COPY . /home/mediacms.io/mediacms From 297df4cd3a0aa97f6b04bec59142d0578bc16d9e Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Mon, 15 Jul 2024 00:57:37 +0300 Subject: [PATCH 15/16] Dockerfile_Alpine image alternative - migrate from ubuntu to alpine - fixing most current vulnerabilities (CVE) - 54% reduce image size with slightly speed change --- Dockerfile_Alpine | 72 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 Dockerfile_Alpine diff --git a/Dockerfile_Alpine b/Dockerfile_Alpine new file mode 100644 index 000000000..036dc087e --- /dev/null +++ b/Dockerfile_Alpine @@ -0,0 +1,72 @@ +FROM python:3.12.4-alpine3.20 AS compile-image + +# Set up virtualenv +ENV VIRTUAL_ENV=/home/mediacms.io +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +ENV PIP_NO_CACHE_DIR=1 + +RUN mkdir -p /home/mediacms.io/mediacms/{logs} && cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV + +RUN apk update && apk upgrade +RUN apk add --no-cache gcc libc-dev linux-headers + +# Install dependencies: +COPY requirements.txt . + +RUN pip install pip==24.1.2 +RUN pip install --upgrade pip +RUN pip install --upgrade setuptools +RUN pip install -r requirements.txt + +COPY . /home/mediacms.io/mediacms +WORKDIR /home/mediacms.io/mediacms + +RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip && \ + unzip Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip -d ../bento4 && \ + mv ../bento4/Bento4-SDK-1-6-0-641.x86_64-unknown-linux/* ../bento4/ && \ + rm -rf ../bento4/Bento4-SDK-1-6-0-641.x86_64-unknown-linux && \ + rm -rf ../bento4/docs && \ + rm Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip + +############ RUNTIME IMAGE ############ +#FROM python:3.13-rc-alpine AS runtime-image +FROM python:alpine AS runtime-image + +ENV PYTHONUNBUFFERED=1 +ENV PYTHONDONTWRITEBYTECODE=1 + +# See: https://github.com/celery/celery/issues/6285#issuecomment-715316219 +ENV CELERY_APP='cms' + +# Use these to toggle which processes supervisord should run +ENV ENABLE_UWSGI='yes' +ENV ENABLE_NGINX='yes' +ENV ENABLE_CELERY_BEAT='yes' +ENV ENABLE_CELERY_SHORT='yes' +ENV ENABLE_CELERY_LONG='yes' +ENV ENABLE_MIGRATIONS='yes' + +# Set up virtualenv +ENV VIRTUAL_ENV=/home/mediacms.io +ENV PATH="$VIRTUAL_ENV/bin:$PATH" + +#RUN pip install --upgrade pip +RUN pip install pip==24.1.2 +#RUN pip uninstall pip==24.0 +RUN pip install certifi + +COPY --chown=www-data:www-data --from=compile-image /home/mediacms.io /home/mediacms.io + +RUN apk update && apk upgrade && apk add --no-cache --no-interactive supervisor nginx imagemagick procps wget xz + +RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \ + mkdir -p ffmpeg-tmp && \ + tar -xf ffmpeg-release-amd64-static.tar.xz --strip-components 1 -C ffmpeg-tmp && \ + cp -v ffmpeg-tmp/ffmpeg ffmpeg-tmp/ffprobe ffmpeg-tmp/qt-faststart /usr/local/bin && \ + rm -rf ffmpeg-tmp ffmpeg-release-amd64-static.tar.xz + +WORKDIR /home/mediacms.io/mediacms +EXPOSE 9000 80 +RUN chmod +x ./deploy/docker/entrypoint.sh +ENTRYPOINT ["./deploy/docker/entrypoint.sh"] +CMD ["./deploy/docker/start.sh"] From b09cb330354db57f600c64930ff534fba5d9f82a Mon Sep 17 00:00:00 2001 From: Laurentiu Date: Thu, 18 Jul 2024 16:31:02 +0300 Subject: [PATCH 16/16] Update Dockerfile_Alpine --- Dockerfile_Alpine | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/Dockerfile_Alpine b/Dockerfile_Alpine index 036dc087e..a28c4b438 100644 --- a/Dockerfile_Alpine +++ b/Dockerfile_Alpine @@ -6,16 +6,11 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH" ENV PIP_NO_CACHE_DIR=1 RUN mkdir -p /home/mediacms.io/mediacms/{logs} && cd /home/mediacms.io && python3 -m venv $VIRTUAL_ENV - RUN apk update && apk upgrade -RUN apk add --no-cache gcc libc-dev linux-headers +RUN apk add --no-cache --virtual .build-deps gcc libc-dev linux-headers # Install dependencies: COPY requirements.txt . - -RUN pip install pip==24.1.2 -RUN pip install --upgrade pip -RUN pip install --upgrade setuptools RUN pip install -r requirements.txt COPY . /home/mediacms.io/mediacms @@ -29,8 +24,7 @@ RUN wget -q http://zebulon.bok.net/Bento4/binaries/Bento4-SDK-1-6-0-641.x86_64-u rm Bento4-SDK-1-6-0-641.x86_64-unknown-linux.zip ############ RUNTIME IMAGE ############ -#FROM python:3.13-rc-alpine AS runtime-image -FROM python:alpine AS runtime-image +FROM python:3.13-rc-alpine AS runtime-image ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 @@ -50,14 +44,9 @@ ENV ENABLE_MIGRATIONS='yes' ENV VIRTUAL_ENV=/home/mediacms.io ENV PATH="$VIRTUAL_ENV/bin:$PATH" -#RUN pip install --upgrade pip -RUN pip install pip==24.1.2 -#RUN pip uninstall pip==24.0 RUN pip install certifi - COPY --chown=www-data:www-data --from=compile-image /home/mediacms.io /home/mediacms.io - -RUN apk update && apk upgrade && apk add --no-cache --no-interactive supervisor nginx imagemagick procps wget xz +RUN apk update && apk upgrade && apk add --no-cache --no-interactive --virtual supervisor nginx imagemagick procps wget xz RUN wget -q https://johnvansickle.com/ffmpeg/releases/ffmpeg-release-amd64-static.tar.xz && \ mkdir -p ffmpeg-tmp && \