Replace django-clamd with django-safe-filefield #TMMA-504

Author: asset-web

browser/models.py

@@ -16,6 +16,7 @@
 from django.utils import timezone
 from django.conf import settings
 
+from safe_filefield.models import SafeFileField
 from mptt.models import MPTTModel, TreeForeignKey
 
 logger = logging.getLogger(__name__)
@@ -140,7 +141,10 @@ class Upload(models.Model):
 
     user = models.ForeignKey(User, null=False, blank=False,
                              related_name="uploads", on_delete=models.CASCADE)
-    abstracts_upload = models.FileField(upload_to=get_user_upload_location)
+    abstracts_upload = SafeFileField(upload_to=get_user_upload_location,
+                                     allowed_extensions=('txt', 'zip', 'bz'),
+                                     check_content_type=True,
+                                     scan_viruses=settings.CLAMD_ENABLED)
     file_format = models.CharField(choices=ABSTRACT_FORMATS, max_length=6, default=OVID)
 
     def __str__(self):

deploy/Dockerfile

@@ -23,6 +23,8 @@ RUN apt update -y && \
             gcc \
             g++ \
             libffi-dev  \
+            # ref: https://github.com/mixkorshun/django-safe-filefield requires libmagic
+            libmagic1 \
             libpq-dev \
             libssl-dev \
             pkg-config \

deploy/deploy-centos.sh

@@ -79,6 +79,9 @@ yum -y install clamd
 # Installed on prod
 yum -y install clamav-data
 yum -y install clamav-devel
+# ref: https://github.com/mixkorshun/django-safe-filefield requires libmagic
+# yum -y install file-devel  # TODO: Review if also required
+yum -y install file-libs
 
 setsebool -P antivirus_can_scan_system 1
 setsebool -P daemons_enable_cluster_mode 1

requirements/dev.txt

@@ -198,7 +198,7 @@ clamd==1.0.2 \
     --hash=sha256:d82a2fd814684a35a1b31feadafb2e69c8ebde9403613f6bdaa5d877c0f29560
     # via
     #   -r requirements/test.txt
-    #   django-clamd
+    #   django-safe-filefield
 click==8.1.7 \
     --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
     --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
@@ -272,18 +272,14 @@ django==4.2.10 \
     # via
     #   -r requirements/test.txt
     #   django-autocomplete-light
-    #   django-clamd
     #   django-debug-toolbar
     #   django-hcaptcha-field
     #   django-js-asset
     #   django-rq
+    #   django-safe-filefield
 django-autocomplete-light==3.9.7 \
     --hash=sha256:a34f192ac438c4df056dbfd399550799ddc631c4661960134ded924648770373
     # via -r requirements/test.txt
-django-clamd==0.4.0 \
-    --hash=sha256:47adaf6f8156ee613dde9e0d88adc847aa3df021dabf1a4d1aec34b47de8da11 \
-    --hash=sha256:5bde6c34b42e69972677fb4a7466b032377940c8727ee6ede41eb3b9807dcce4
-    # via -r requirements/test.txt
 django-debug-toolbar==4.3.0 \
     --hash=sha256:0b0dddee5ea29b9cb678593bc0d7a6d76b21d7799cb68e091a2148341a80f3c4 \
     --hash=sha256:e09b7dcb8417b743234dfc57c95a7c1d1d87a88844abd13b4c5387f807b31bf6
@@ -313,6 +309,10 @@ django-rq==2.10.1 \
     --hash=sha256:c69af7b2b7d5745847cdaa80c58b70e564ea81c90712a11362d0bd8c71f5ce94 \
     --hash=sha256:cf588efe248e275ef176190a7db49a33281049ff3857153577c2c49320bfd4bb
     # via -r requirements/test.txt
+django-safe-filefield==1.0.0 \
+    --hash=sha256:d08b0034e845ba78d2ab6e144b305bb5d7ee4a999c14861de22f8dc905a1ffe2 \
+    --hash=sha256:f27b7770693b28716a5db8bd81f83b9bd501067284f6b69bf67a9ca4f21e415d
+    # via -r requirements/test.txt
 easyprocess==1.1 \
     --hash=sha256:82eed523a0a5eb12a81fa4eacd9f342caeb3f900eb4b798740e6696ad07e63f9 \
     --hash=sha256:885898302a57aab948973e8b5d32a4229392b9fb2d986ab1d4ffd590e5ba90ec
@@ -760,6 +760,7 @@ python-magic==0.4.27 \
     --hash=sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3
     # via
     #   -r requirements/test.txt
+    #   django-safe-filefield
     #   xtract
 pytz==2024.1 \
     --hash=sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812 \

requirements/requirements.in

@@ -1,14 +1,14 @@
 # This is an implicit value, here for clarity
 --index-url https://pypi.python.org/simple/
 Bottleneck>=1.3.6
-django-clamd>=0.4.0
 django-mptt>=0.14.0
 django-redis-cache>=3.0.1
 django-registration-redux>=2.12
 django-rq>=2.7.0
 django-autocomplete-light>=3.9.4
 Django<4.3
 django-hcaptcha-field>=1.4.0
+django-safe-filefield>=1.0.0
 hiredis>=2.2.2
 lxml>=4.9.2
 more-itertools>=9.1.0

requirements/requirements.txt

@@ -76,7 +76,7 @@ bottleneck==1.3.7 \
 clamd==1.0.2 \
     --hash=sha256:5c32546b7d1eb00fd6be00a889d79e00fbf980ed082826ccfa369bce3dcff5e7 \
     --hash=sha256:d82a2fd814684a35a1b31feadafb2e69c8ebde9403613f6bdaa5d877c0f29560
-    # via django-clamd
+    # via django-safe-filefield
 click==8.1.7 \
     --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
     --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
@@ -87,17 +87,13 @@ django==4.2.10 \
     # via
     #   -r requirements/requirements.in
     #   django-autocomplete-light
-    #   django-clamd
     #   django-hcaptcha-field
     #   django-js-asset
     #   django-rq
+    #   django-safe-filefield
 django-autocomplete-light==3.9.7 \
     --hash=sha256:a34f192ac438c4df056dbfd399550799ddc631c4661960134ded924648770373
     # via -r requirements/requirements.in
-django-clamd==0.4.0 \
-    --hash=sha256:47adaf6f8156ee613dde9e0d88adc847aa3df021dabf1a4d1aec34b47de8da11 \
-    --hash=sha256:5bde6c34b42e69972677fb4a7466b032377940c8727ee6ede41eb3b9807dcce4
-    # via -r requirements/requirements.in
 django-hcaptcha-field==1.4.0 \
     --hash=sha256:36001030f53af709db3be01bf1d25023706b701bb7130f67f2aa58900b20df42 \
     --hash=sha256:bcf2f698b1dc5f8ce411c11696df5785298df0cbd2fa564b09bd96609060bf34
@@ -121,6 +117,10 @@ django-rq==2.10.1 \
     --hash=sha256:c69af7b2b7d5745847cdaa80c58b70e564ea81c90712a11362d0bd8c71f5ce94 \
     --hash=sha256:cf588efe248e275ef176190a7db49a33281049ff3857153577c2c49320bfd4bb
     # via -r requirements/requirements.in
+django-safe-filefield==1.0.0 \
+    --hash=sha256:d08b0034e845ba78d2ab6e144b305bb5d7ee4a999c14861de22f8dc905a1ffe2 \
+    --hash=sha256:f27b7770693b28716a5db8bd81f83b9bd501067284f6b69bf67a9ca4f21e415d
+    # via -r requirements/requirements.in
 hiredis==2.3.2 \
     --hash=sha256:01b6c24c0840ac7afafbc4db236fd55f56a9a0919a215c25a238f051781f4772 \
     --hash=sha256:02fc71c8333586871602db4774d3a3e403b4ccf6446dc4603ec12df563127cee \
@@ -457,6 +457,7 @@ python-magic==0.4.27 \
     --hash=sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3
     # via
     #   -r requirements/requirements.in
+    #   django-safe-filefield
     #   xtract
 pytz==2024.1 \
     --hash=sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812 \

requirements/test.txt

@@ -186,7 +186,7 @@ clamd==1.0.2 \
     --hash=sha256:d82a2fd814684a35a1b31feadafb2e69c8ebde9403613f6bdaa5d877c0f29560
     # via
     #   -r requirements/requirements.txt
-    #   django-clamd
+    #   django-safe-filefield
 click==8.1.7 \
     --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \
     --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de
@@ -256,17 +256,13 @@ django==4.2.10 \
     # via
     #   -r requirements/requirements.txt
     #   django-autocomplete-light
-    #   django-clamd
     #   django-hcaptcha-field
     #   django-js-asset
     #   django-rq
+    #   django-safe-filefield
 django-autocomplete-light==3.9.7 \
     --hash=sha256:a34f192ac438c4df056dbfd399550799ddc631c4661960134ded924648770373
     # via -r requirements/requirements.txt
-django-clamd==0.4.0 \
-    --hash=sha256:47adaf6f8156ee613dde9e0d88adc847aa3df021dabf1a4d1aec34b47de8da11 \
-    --hash=sha256:5bde6c34b42e69972677fb4a7466b032377940c8727ee6ede41eb3b9807dcce4
-    # via -r requirements/requirements.txt
 django-hcaptcha-field==1.4.0 \
     --hash=sha256:36001030f53af709db3be01bf1d25023706b701bb7130f67f2aa58900b20df42 \
     --hash=sha256:bcf2f698b1dc5f8ce411c11696df5785298df0cbd2fa564b09bd96609060bf34
@@ -292,6 +288,10 @@ django-rq==2.10.1 \
     --hash=sha256:c69af7b2b7d5745847cdaa80c58b70e564ea81c90712a11362d0bd8c71f5ce94 \
     --hash=sha256:cf588efe248e275ef176190a7db49a33281049ff3857153577c2c49320bfd4bb
     # via -r requirements/requirements.txt
+django-safe-filefield==1.0.0 \
+    --hash=sha256:d08b0034e845ba78d2ab6e144b305bb5d7ee4a999c14861de22f8dc905a1ffe2 \
+    --hash=sha256:f27b7770693b28716a5db8bd81f83b9bd501067284f6b69bf67a9ca4f21e415d
+    # via -r requirements/requirements.txt
 easyprocess==1.1 \
     --hash=sha256:82eed523a0a5eb12a81fa4eacd9f342caeb3f900eb4b798740e6696ad07e63f9 \
     --hash=sha256:885898302a57aab948973e8b5d32a4229392b9fb2d986ab1d4ffd590e5ba90ec
@@ -675,6 +675,7 @@ python-magic==0.4.27 \
     --hash=sha256:c212960ad306f700aa0d01e5d7a325d20548ff97eb9920dcd29513174f0294d3
     # via
     #   -r requirements/requirements.txt
+    #   django-safe-filefield
     #   xtract
 pytz==2024.1 \
     --hash=sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812 \

temmpo/settings/base.py

@@ -62,7 +62,7 @@
     'django.contrib.humanize',
 ]
 
-THIRD_PARTY_APPS = ['registration', 'mptt', 'django_rq', 'django_clamd', 'hcaptcha_field']
+THIRD_PARTY_APPS = ['registration', 'mptt', 'django_rq', 'safe_filefield', 'hcaptcha_field']
 LOCAL_APPS = ['browser', 'dal', 'dal_select2',] # 'dal','dal_select2' are third party apps that need to be installed before 'django.contrib.admin'
 
 INSTALLED_APPS = LOCAL_APPS + DEFAULT_APPS + THIRD_PARTY_APPS
@@ -232,11 +232,11 @@
         },
     },
 }
-
-CLAMD_SOCKET = '/var/run/clamd.scan/clamd.sock'
-CLAMD_USE_TCP = False
-CLAMD_TCP_SOCKET = 3310
-CLAMD_TCP_ADDR = '127.0.0.1'
+# ref: https://github.com/mixkorshun/django-safe-filefield
+CLAMAV_SOCKET = '/var/run/clamd.scan/clamd.sock'
+# CLAMAV_TIMEOUT 
+# Use to disable scanning of files in environments where it is not supported, 
+# e.g. a bare bones GitHub action environment for example
 CLAMD_ENABLED = True
 
 DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'