diff --git a/.gitignore b/.gitignore index 8d35cb3..c8928da 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ __pycache__ *.pyc +django_byostorage.egg-info diff --git a/byostorage/cached.py b/byostorage/cached.py new file mode 100644 index 0000000..87fde0d --- /dev/null +++ b/byostorage/cached.py @@ -0,0 +1,100 @@ +from django.core.files.storage import Storage, get_storage_class +from django.utils.deconstruct import deconstructible +from django.conf import settings +from hashlib import sha1 +import os.path +import shutil +import tempfile +import time + +import logging +logger = logging.getLogger(__name__) + +@deconstructible +class CachedStorage(Storage): + + CACHE_EXPIRES = 30 + + def __init__(self, remote=None, cachedir=None): + if not remote: + remote = settings.CACHED_STORAGE_REMOTE + if not cachedir: + cachedir = settings.CACHED_STORAGE_DIR + self.remote = get_storage_class(remote)() + self.cachedir = cachedir + os.makedirs(self.cachedir, exist_ok=True) + self.clean() + + def _filepath(self, name): + base, ext = os.path.splitext(name) + filename = sha1(base.encode('utf8')).hexdigest() + ext + return os.path.join(self.cachedir, filename) + + def _cached(self, name): + p = self._filepath(name) + if not os.path.exists(p): + logger.debug("Caching %s to %s", name, p) + source = self.remote.open(name, 'rb') + dest = tempfile.NamedTemporaryFile(dir=self.cachedir, delete=False, prefix="_") + shutil.copyfileobj(source, dest) + source.close() + dest.close() + os.rename(dest.name, p) + now = time.time() + os.utime(p, (now, now)) + + if now > self.next_check: + self.clean() # wont get this file as we just touched it + + return p + + + def _open(self, name, mode='rb'): + assert 'r' in mode, "Can only open for reading" + p = self._cached(name) + return open(p, mode) + + def path(self, name): + return self._cached(name) + + def save(self, name, content, max_length=None): + p = self._filepath(name) + if os.path.exists(p): + os.unlink(p) + return self.remote.save(name, content, max_length) + + def delete(self, name): + return self.remote.delete(name) + + def exists(self, name): + return self.remote.exists(name) + + def listdir(self, name): + return self.remote.listdir(name) + + def size(self, name): + return self.remote.size(name) + + def url(self, name): + return self.remote.url(name) + + def get_valid_name(self, name): + return self.remote.get_valid_name(name) + + def get_available_name(self, name, max_length=None): + return self.remote.get_available_name(name, max_length) + + def get_alternative_name(self, file_root, file_ext): + return self.remote.get_alternative_name(file_root, file_ext) + + def clean(self): + now = time.time() + threshold = now - self.CACHE_EXPIRES + logger.info("Removing cached files older than %d seconds", self.CACHE_EXPIRES) + for f in os.listdir(self.cachedir): + f = os.path.join(self.cachedir, f) + s = os.stat(f) + if s.st_atime < threshold: + logger.debug("Removing %s", f) + os.unlink(f) + self.next_check = now + 300 diff --git a/byostorage/migrations/0001_initial.py b/byostorage/migrations/0001_initial.py index 13406a4..c585f0e 100644 --- a/byostorage/migrations/0001_initial.py +++ b/byostorage/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.1.1 on 2021-03-11 23:11 +# Generated by Django 3.1.7 on 2021-03-11 22:07 from django.conf import settings from django.db import migrations, models @@ -18,8 +18,8 @@ class Migration(migrations.Migration): name='UserStorage', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('name', models.SlugField(help_text='Storage tag', max_length=20)), - ('storage', models.CharField(help_text='Storage class for this instance', max_length=255)), + ('name', models.SlugField(help_text='Storage tag', max_length=20, unique=True)), + ('storage', models.CharField(choices=[('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage')], help_text='Storage class for this instance', max_length=255)), ('settings_data', models.TextField(help_text='JSON dict with key/value settings')), ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], diff --git a/byostorage/migrations/0002_auto_20210311_1826.py b/byostorage/migrations/0002_auto_20210311_1826.py deleted file mode 100644 index 59c864b..0000000 --- a/byostorage/migrations/0002_auto_20210311_1826.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 3.1.7 on 2021-03-11 18:26 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('byostorage', '0001_initial'), - ] - - operations = [ - migrations.AlterField( - model_name='userstorage', - name='storage', - field=models.CharField(choices=[], help_text='Storage class for this instance', max_length=255), - ), - ] diff --git a/byostorage/migrations/0003_auto_20210311_1917.py b/byostorage/migrations/0003_auto_20210311_1917.py deleted file mode 100644 index e38a1dc..0000000 --- a/byostorage/migrations/0003_auto_20210311_1917.py +++ /dev/null @@ -1,23 +0,0 @@ -# Generated by Django 3.1.7 on 2021-03-11 19:17 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('byostorage', '0002_auto_20210311_1826'), - ] - - operations = [ - migrations.AlterField( - model_name='userstorage', - name='name', - field=models.SlugField(help_text='Storage tag', max_length=20, unique=True), - ), - migrations.AlterField( - model_name='userstorage', - name='storage', - field=models.CharField(choices=[('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage')], help_text='Storage class for this instance', max_length=255), - ), - ] diff --git a/byostorage/storage.py b/byostorage/multi.py similarity index 80% rename from byostorage/storage.py rename to byostorage/multi.py index 755eb51..d56f497 100644 --- a/byostorage/storage.py +++ b/byostorage/multi.py @@ -1,16 +1,20 @@ from django.core.files.storage import Storage from django.conf import settings +from django.utils.deconstruct import deconstructible ''' TODO: Create a signal to remove instances from cache if modified ''' +@deconstructible class MultiStorage(Storage): ''' Django storage class that proxies multiple storage classes. Uses a name prefix to determine store e.g. 'images:foo/bar.jpg' ''' + DELIM = ":" + def __init__(self, config=None): self.config = config or getattr(settings, 'BRING_YOUR_OWN_STORAGE', {}) self._cache = {} @@ -22,7 +26,14 @@ class MultiStorage(Storage): return self._cache[name] def split(self, name): - return name.split("/", 1) + try: + storage, p = name.split(self.DELIM, 1) + return storage, p + except ValueError: + raise ValueError(f"Bad storage path [{name}]") + + def join(self, storage, name): + return self.DELIM.join((storage, name)) def _proxy(self, method, name, *args, **kwargs): print('PROXY', method, name, *args, **kwargs) @@ -36,13 +47,13 @@ class MultiStorage(Storage): storage, p = self.split(name) result = getattr(self.get_storage(storage), method)(p, *args, **kwargs) print("RESULT", result) - return f"{storage}/{result}" + return self.join(storage, result) def _open(self, name, mode='rb'): - return self._proxy('_open', name, mode) + return self._proxy('open', name, mode) - def _save(self, name, content, max_length=None): - return self._proxy_name('_save', name, content, max_length) + def save(self, name, content, max_length=None): + return self._proxy_name('save', name, content, max_length) def exists(self, name): return self._proxy('exists', name) @@ -65,8 +76,6 @@ class MultiStorage(Storage): def get_modified_time(self, name): return self._proxy('get_modified_time', name) - def get_valid_name(self, name): - return self._proxy_name('get_valid_name', name) class BYOStorage(MultiStorage): ''' Database driven Bring-Your-Own-Storage