First working version

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 *.pyc
+django_byostorage.egg-info

byostorage/cached.py

@@ -0,0 +1,100 @@
+from django.core.files.storage import Storage, get_storage_class
+from django.utils.deconstruct import deconstructible
+from django.conf import settings
+from hashlib import sha1
+import os.path
+import shutil
+import tempfile
+import time
+
+import logging
+logger = logging.getLogger(__name__)
+
+@deconstructible
+class CachedStorage(Storage):
+    
+    CACHE_EXPIRES = 30
+
+    def __init__(self, remote=None, cachedir=None):
+        if not remote:
+            remote = settings.CACHED_STORAGE_REMOTE
+        if not cachedir:
+            cachedir = settings.CACHED_STORAGE_DIR
+        self.remote = get_storage_class(remote)()
+        self.cachedir = cachedir
+        os.makedirs(self.cachedir, exist_ok=True)
+        self.clean()
+
+    def _filepath(self, name):
+        base, ext = os.path.splitext(name)
+        filename = sha1(base.encode('utf8')).hexdigest() + ext
+        return os.path.join(self.cachedir, filename)
+
+    def _cached(self, name):
+        p = self._filepath(name)
+        if not os.path.exists(p):
+            logger.debug("Caching %s to %s", name, p)
+            source = self.remote.open(name, 'rb')
+            dest = tempfile.NamedTemporaryFile(dir=self.cachedir, delete=False, prefix="_")
+            shutil.copyfileobj(source, dest)
+            source.close()
+            dest.close()
+            os.rename(dest.name, p)
+        now = time.time()
+        os.utime(p, (now, now))
+
+        if now > self.next_check:
+            self.clean() # wont get this file as we just touched it
+
+        return p
+
+
+    def _open(self, name, mode='rb'):
+        assert 'r' in mode, "Can only open for reading"
+        p = self._cached(name)
+        return open(p, mode)
+
+    def path(self, name):
+        return self._cached(name)
+
+    def save(self, name, content, max_length=None):
+        p = self._filepath(name)
+        if os.path.exists(p):
+            os.unlink(p)
+        return self.remote.save(name, content, max_length)
+
+    def delete(self, name):
+        return self.remote.delete(name)
+
+    def exists(self, name):
+        return self.remote.exists(name)
+
+    def listdir(self, name):
+        return self.remote.listdir(name)
+
+    def size(self, name):
+        return self.remote.size(name)
+
+    def url(self, name):
+        return self.remote.url(name)
+
+    def get_valid_name(self, name):
+        return self.remote.get_valid_name(name)
+
+    def get_available_name(self, name, max_length=None):
+        return self.remote.get_available_name(name, max_length)
+
+    def get_alternative_name(self, file_root, file_ext):
+        return self.remote.get_alternative_name(file_root, file_ext)
+
+    def clean(self):
+        now = time.time()
+        threshold = now - self.CACHE_EXPIRES
+        logger.info("Removing cached files older than %d seconds", self.CACHE_EXPIRES)
+        for f in os.listdir(self.cachedir):
+            f = os.path.join(self.cachedir, f)
+            s = os.stat(f)
+            if s.st_atime < threshold:
+                logger.debug("Removing %s", f)
+                os.unlink(f)
+        self.next_check = now + 300

byostorage/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.1.1 on 2021-03-11 23:11
+# Generated by Django 3.1.7 on 2021-03-11 22:07
 
 from django.conf import settings
 from django.db import migrations, models
@@ -18,8 +18,8 @@ class Migration(migrations.Migration):
             name='UserStorage',
             fields=[
                 ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
-                ('name', models.SlugField(help_text='Storage tag', max_length=20)),
+                ('name', models.SlugField(help_text='Storage tag', max_length=20, unique=True)),
-                ('storage', models.CharField(help_text='Storage class for this instance', max_length=255)),
+                ('storage', models.CharField(choices=[('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage')], help_text='Storage class for this instance', max_length=255)),
                 ('settings_data', models.TextField(help_text='JSON dict with key/value settings')),
                 ('owner', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],

byostorage/migrations/0002_auto_20210311_1826.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.1.7 on 2021-03-11 18:26
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('byostorage', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='userstorage',
-            name='storage',
-            field=models.CharField(choices=[], help_text='Storage class for this instance', max_length=255),
-        ),
-    ]

byostorage/migrations/0003_auto_20210311_1917.py

@@ -1,23 +0,0 @@
-# Generated by Django 3.1.7 on 2021-03-11 19:17
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('byostorage', '0002_auto_20210311_1826'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='userstorage',
-            name='name',
-            field=models.SlugField(help_text='Storage tag', max_length=20, unique=True),
-        ),
-        migrations.AlterField(
-            model_name='userstorage',
-            name='storage',
-            field=models.CharField(choices=[('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage')], help_text='Storage class for this instance', max_length=255),
-        ),
-    ]

byostorage/multi.py

@@ -1,16 +1,20 @@
 from django.core.files.storage import Storage
 from django.conf import settings
+from django.utils.deconstruct import deconstructible
 
 '''
 TODO: Create a signal to remove instances from cache if modified
 '''
 
+@deconstructible
 class MultiStorage(Storage):
     ''' Django storage class that proxies multiple storage classes.
 
     Uses a name prefix to determine store e.g. 'images:foo/bar.jpg'
     '''
 
+    DELIM = ":"
+
     def __init__(self, config=None):
         self.config = config or getattr(settings, 'BRING_YOUR_OWN_STORAGE', {})
         self._cache = {}
@@ -22,7 +26,14 @@ class MultiStorage(Storage):
         return self._cache[name]
 
     def split(self, name):
-        return name.split("/", 1)
+        try:
+            storage, p = name.split(self.DELIM, 1)
+            return storage, p
+        except ValueError:
+            raise ValueError(f"Bad storage path [{name}]")
+
+    def join(self, storage, name):
+        return self.DELIM.join((storage, name))
 
     def _proxy(self, method, name, *args, **kwargs):
         print('PROXY', method, name, *args, **kwargs)
@@ -36,13 +47,13 @@ class MultiStorage(Storage):
         storage, p = self.split(name)
         result = getattr(self.get_storage(storage), method)(p, *args, **kwargs)
         print("RESULT", result)
-        return f"{storage}/{result}"
+        return self.join(storage, result)
 
     def _open(self, name, mode='rb'):
-        return self._proxy('_open', name, mode)
+        return self._proxy('open', name, mode)
 
-    def _save(self, name, content, max_length=None):
+    def save(self, name, content, max_length=None):
-        return self._proxy_name('_save', name, content, max_length)
+        return self._proxy_name('save', name, content, max_length)
 
     def exists(self, name):
         return self._proxy('exists', name)
@@ -65,8 +76,6 @@ class MultiStorage(Storage):
     def get_modified_time(self, name):
         return self._proxy('get_modified_time', name)
 
-    def get_valid_name(self, name):
-        return self._proxy_name('get_valid_name', name)
 
 class BYOStorage(MultiStorage):
     ''' Database driven Bring-Your-Own-Storage