from django.core.files.storage import Storage, DefaultStorage from django.utils.deconstruct import deconstructible from django.utils.module_loading import import_string from django.conf import settings from hashlib import sha1 import os.path import shutil import tempfile import time import logging logger = logging.getLogger(__name__) @deconstructible class CachedStorage(Storage): def __init__(self, remote=None, cachedir=None, expires=600): if not remote: remote = getattr(settings, 'CACHED_STORAGE_REMOTE', DefaultStorage) if not cachedir: cachedir = getattr(settings, 'CACHED_STORAGE_DIR', 'cache') if isinstance(remote, Storage): self.remote = remote else: self.remote = import_string(remote)() self.cachedir = cachedir self.expires = expires os.makedirs(self.cachedir, exist_ok=True) self.clean() def _filepath(self, name): base, ext = os.path.splitext(name) filename = sha1(base.encode('utf8')).hexdigest() + ext return os.path.join(self.cachedir, filename) def _cached(self, name): p = self._filepath(name) if not os.path.exists(p): logger.debug("Caching %s to %s", name, p) source = self.remote.open(name, 'rb') dest = tempfile.NamedTemporaryFile(dir=self.cachedir, delete=False, prefix="_") shutil.copyfileobj(source, dest) source.close() dest.close() os.rename(dest.name, p) now = time.time() os.utime(p, (now, now)) if now > self.next_check: self.clean() # wont get this file as we just touched it return p def _open(self, name, mode='rb'): assert 'r' in mode, "Can only open for reading" p = self.path(name) return open(p, mode) def path(self, name): try: return self.remote.path(name) except NotImplementedError: return self._cached(name) def save(self, name, content, max_length=None): p = self._filepath(name) if os.path.exists(p): os.unlink(p) # TODO: cache content to p return self.remote.save(name, content, max_length) def delete(self, name): return self.remote.delete(name) def exists(self, name): return self.remote.exists(name) def listdir(self, name): return self.remote.listdir(name) def size(self, name): return self.remote.size(name) def url(self, name): return self.remote.url(name) def get_valid_name(self, name): return self.remote.get_valid_name(name) def get_available_name(self, name, max_length=None): return self.remote.get_available_name(name, max_length) def get_alternative_name(self, file_root, file_ext): return self.remote.get_alternative_name(file_root, file_ext) def clean(self): now = time.time() threshold = now - self.expires logger.info("Removing cached files older than %d seconds", self.expires) for f in os.listdir(self.cachedir): f = os.path.join(self.cachedir, f) s = os.stat(f) if s.st_atime < threshold: logger.debug("Removing %s", f) os.unlink(f) self.next_check = now + 300 # wait at least 5 minutes before running again