Add cache for files fetch all metadata in single request

README.rst

@@ -121,6 +121,8 @@ django-storage-swift recognises the following options.
 +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+
 | ``SWIFT_CACHE_HEADERS``                      | False          | Headers cache on/off switcher                                                                                                                      |
 +----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+
+| ``SWIFT_FILE_CACHE``                         | False          | Enable files metadata cache, before start collectstatic client request all files metadata, in a single request.                                    |
++----------------------------------------------+----------------+----------------------------------------------------------------------------------------------------------------------------------------------------+
 
 
 SWIFT\_BASE\_URL

swift/storage.py

@@ -3,6 +3,7 @@
 import os
 import re
 from datetime import datetime
+from email.utils import parsedate_to_datetime
 from functools import wraps
 from io import BytesIO, UnsupportedOperation
 from time import time
@@ -12,6 +13,7 @@
 from django.core.files import File
 from django.core.files.storage import Storage
 from six.moves.urllib import parse as urlparse
+import pytz
 
 try:
     from django.utils.deconstruct import deconstructible
@@ -29,7 +31,6 @@ def deconstructible(arg):
 def setting(name, default=None):
     return getattr(settings, name, default)
 
-
 def validate_settings(backend):
     # Check mandatory parameters
     if not backend.api_auth_url:
@@ -155,6 +156,8 @@ class SwiftStorage(Storage):
     full_listing = setting('SWIFT_FULL_LISTING', True)
     max_retries = setting('SWIFT_MAX_RETRIES', 5)
     cache_headers = setting('SWIFT_CACHE_HEADERS', False)
+    file_cache_enabled = setting('SWIFT_FILE_CACHE', False)
+    file_cache = None
 
     def __init__(self, **settings):
         # check if some of the settings provided as class attributes
@@ -242,6 +245,30 @@ def base_url(self):
                 self._base_url = self.override_base_url
         return self._base_url
 
+    def build_file_cache(self):
+        if(self.file_cache_enabled and self.file_cache == None):
+            # Build cache if never built
+            data = self.swift_conn.get_container(
+                self.container_name,
+                prefix=self.name_prefix,
+                full_listing=self.full_listing
+            )
+
+            self.file_cache = {}
+            for el in data[1]:
+                ctime_str = el['last_modified']
+                name = el['name']
+                ctime_dt = datetime.strptime(ctime_str, "%Y-%m-%dT%H:%M:%S.%f").replace(tzinfo=pytz.utc)
+                hash = el['hash']
+
+                if settings.USE_TZ:
+                    last_modified = ctime_dt
+                else:
+                    raise ImproperlyConfigured("settings.USE_TZ cannot be False when using swiftclient")
+
+                self.file_cache[name] = {'last_modified': last_modified, 'hash': hash }
+
+
     def _open(self, name, mode='rb'):
         original_name = name
         name = self.name_prefix + name
@@ -322,6 +349,14 @@ def get_headers(self, name):
 
     @prepend_name_prefix
     def exists(self, name):
+        if(self.file_cache_enabled):
+            self.build_file_cache()
+            try:
+                self.file_cache[name]
+            except KeyError:
+                return False
+            return True
+
         try:
             self.get_headers(name)
         except swiftclient.ClientException:
@@ -365,11 +400,6 @@ def get_available_name(self, name, max_length=None):
     def size(self, name):
         return int(self.get_headers(name)['content-length'])
 
-    @prepend_name_prefix
-    def modified_time(self, name):
-        return datetime.fromtimestamp(
-            float(self.get_headers(name)['x-timestamp']))
-
     @prepend_name_prefix
     def url(self, name):
         return self._path(name)
@@ -390,9 +420,6 @@ def _path(self, name):
 
         return url
 
-    def path(self, name):
-        raise NotImplementedError
-
     @prepend_name_prefix
     def isdir(self, name):
         return '.' not in name
@@ -428,7 +455,26 @@ def rmtree(self, abs_path):
             if obj['name'].startswith(abs_path):
                 self.swift_conn.delete_object(self.container_name,
                                               obj['name'])
+    @prepend_name_prefix
+    def get_accessed_time(self, name):
+        # Swift does not get this info, left NotImplemented
+        raise NotImplementedError('subclasses of Storage must provide a get_accessed_time() method')
 
+    @prepend_name_prefix
+    def get_created_time(self, name):
+        logger.debug("get_reated_time")
+        return datetime.fromtimestamp(
+            float(self.get_headers(name)['x-timestamp']))
+
+    @prepend_name_prefix
+    def get_modified_time(self, name):
+        # Handle case when a cache ready
+        if(self.file_cache_enabled):
+            date_timezone = self.file_cache[name]['last_modified']
+        else:
+            data_str = self.get_headers(name)['last-modified']
+            date_timezone = parsedate_to_datetime(data_str)
+        return date_timezone
 
 class StaticSwiftStorage(SwiftStorage):
     container_name = setting('SWIFT_STATIC_CONTAINER_NAME', '')