Updated cache to not use get_storage_class

.gitignore

@@ -1,3 +1,6 @@
 __pycache__
 *.pyc
 django_byostorage.egg-info
+.coverage
+htmlcov
+db.sqlite3

README.rst

@@ -23,3 +23,8 @@ Usage
      photo = models.FileField(
          storage=BYOStorage(),
          upload_to=generate_filename)
+
+Notes
+=====
+
+Boto3 client sometimes doesn't respect the `region_name` so it may be neccesary to set `endpoint_url: https://https://s3-<region>.amazonaws.com`

byostorage/__init__.py

@@ -1,2 +1,2 @@
-__version__ = 0.1
+__version__ = '0.2.4'
 default_app_config = 'byostorage.apps.BYOStorageConfig'

byostorage/apps.py

@@ -4,3 +4,4 @@ from django.apps import AppConfig
 class BYOStorageConfig(AppConfig):
     name = 'byostorage'
     verbose_name = 'Bring your own storage'
+    default_auto_field = 'django.db.models.AutoField'

byostorage/cached.py

@@ -1,5 +1,6 @@
-from django.core.files.storage import Storage, get_storage_class
+from django.core.files.storage import Storage, DefaultStorage
 from django.utils.deconstruct import deconstructible
+from django.utils.module_loading import import_string
 from django.conf import settings
 from hashlib import sha1
 import os.path
@@ -15,11 +16,14 @@ class CachedStorage(Storage):
     
     def __init__(self, remote=None, cachedir=None, expires=600):
         if not remote:
-            remote = getattr(settings, 'CACHED_STORAGE_REMOTE', None)
+            remote = getattr(settings, 'CACHED_STORAGE_REMOTE', DefaultStorage)
         if not cachedir:
             cachedir = getattr(settings, 'CACHED_STORAGE_DIR', 'cache')
 
-        self.remote = get_storage_class(remote)()
+        if isinstance(remote, Storage):
+            self.remote = remote
+        else:
+            self.remote = import_string(remote)()
         self.cachedir = cachedir
         self.expires = expires
 
@@ -52,16 +56,20 @@ class CachedStorage(Storage):
 
     def _open(self, name, mode='rb'):
         assert 'r' in mode, "Can only open for reading"
-        p = self._cached(name)
+        p = self.path(name)
         return open(p, mode)
 
     def path(self, name):
-        return self._cached(name)
+        try:
+            return self.remote.path(name)
+        except NotImplementedError:
+            return self._cached(name)
 
     def save(self, name, content, max_length=None):
         p = self._filepath(name)
         if os.path.exists(p):
             os.unlink(p)
+        # TODO: cache content to p
         return self.remote.save(name, content, max_length)
 
     def delete(self, name):

byostorage/http.py

@@ -0,0 +1,26 @@
+from django.core.files.storage import Storage
+import requests
+
+class HTTPStorage(Storage):
+
+    def __init__(self, protocol='https'):
+        self.protocol = protocol
+
+    def _open(self, name, mode='rb'):
+        if mode != 'rb':
+            raise IOError("Mode must be 'rb'")
+
+        r = requests.get(self.url(name), stream=True)
+        r.raise_for_status()
+
+        return r.raw
+
+    def _save(self, name, content):
+        raise NotImplementedError("Unable to save to web locations")
+
+    def url(self, name):
+        return f"{self.protocol}:{name}"
+
+    def exists(self, name):
+        r = requests.head(self.url(name))
+        return r.status_code == 200

byostorage/local_settings.py

@@ -1,6 +1,24 @@
+from pathlib import Path
+
+# Build paths inside the project like this: BASE_DIR / 'subdir'.
+BASE_DIR = Path(__file__).resolve().parent.parent
+
+MEDIA_ROOT = "media"
+
+DEBUG = True
+DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
+
 INSTALLED_APPS = [
     'django.contrib.auth',
     'django.contrib.contenttypes',
     'byostorage'
 ]
+
 SECRET_KEY = 'shh!'
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': BASE_DIR / 'db.sqlite3',
+    }
+}

byostorage/migrations/0003_auto_20210323_1047.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.1.1 on 2021-03-22 23:47
+
+import byostorage.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0002_auto_20210312_1712'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userstorage',
+            name='settings_data',
+            field=models.TextField(help_text='JSON dict with key/value settings', validators=[byostorage.models.validate_json]),
+        ),
+    ]

byostorage/migrations/0004_auto_20210917_0000.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.2.7 on 2021-09-17 00:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0003_auto_20210323_1047'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='userstorage',
+            name='id',
+        ),
+        migrations.AlterField(
+            model_name='userstorage',
+            name='name',
+            field=models.SlugField(help_text='Storage tag', max_length=20, primary_key=True, serialize=False),
+        ),
+    ]

byostorage/migrations/0005_alter_userstorage_storage.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.7 on 2022-09-09 23:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0004_auto_20210917_0000'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userstorage',
+            name='storage',
+            field=models.CharField(choices=[('django.core.files.storage.FileSystemStorage', 'django.core.files.storage.FileSystemStorage')], help_text='Storage class for this instance', max_length=255),
+        ),
+    ]

byostorage/migrations/0006_alter_userstorage_settings_data.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.7 on 2022-09-10 00:10
+
+import byostorage.models
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('byostorage', '0005_alter_userstorage_storage'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='userstorage',
+            name='settings_data',
+            field=models.TextField(default='{}', help_text='JSON dict with key/value settings', validators=[byostorage.models.validate_json]),
+        ),
+    ]

byostorage/models.py

@@ -1,8 +1,16 @@
 from django.conf import settings
 from django.db import models
-from django.core.files.storage import get_storage_class
+from django.db.models.fields.files import FieldFile
+from django.core.exceptions import ObjectDoesNotExist
+from django.core.files.storage import Storage
+from django.core.exceptions import ValidationError
+from django.utils.module_loading import import_string
 import json
 
+import logging
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
 STORAGE_CLASSES = getattr(settings, 'STORAGE_CLASSES', [])
 
 STORAGE_CLASSES.extend([
@@ -10,13 +18,36 @@ STORAGE_CLASSES.extend([
 ])
 
 try:
-    import storages
+    import storages.backends
-    STORAGE_CLASSES.append('storages.backends.s3boto3.S3Boto3Storage')
+
+    STORAGE_CLASSES.extend([
+        'storages.backends.s3boto3.S3Boto3Storage',
+        'storages.backends.azure_storage.AzureStorage',
+        'storages.backends.dropbox.DropBoxStorage',
+        'storages.backends.ftp.FTPStorage',
+        'storages.backends.gcloud.GoogleCloudStorage',
+        'storages.backends.sftpstorage.SFTPStorage',
+    ])
 except ImportError:
     pass
 
 STORAGE_CLASS_OPTIONS = [ (x, x) for x in STORAGE_CLASSES ]
 
+def validate_json(value):
+    try:
+        json.loads(value)
+    except json.JSONDecodeError as e:
+        raise ValidationError(e.msg)
+
+def resolve_instance(obj, relation):
+    for p in relation.split('__'):
+        obj = getattr(obj, p)
+
+        if callable(obj):
+            obj = obj()
+
+    return obj
+
 class UserStorage(models.Model):
     """ A user defined storage
     """
@@ -24,26 +55,90 @@ class UserStorage(models.Model):
             on_delete=models.CASCADE,
             null=True, blank=True)
     name = models.SlugField(max_length=20,
-            unique=True,
+            primary_key=True,
             help_text="Storage tag")
     storage = models.CharField(max_length=255,
             choices=STORAGE_CLASS_OPTIONS,
             help_text="Storage class for this instance")
-    settings_data = models.TextField(help_text="JSON dict with key/value settings")
+    settings_data = models.TextField(
+            validators=[validate_json],
+            default='{}',
+            help_text="JSON dict with key/value settings")
 
     def instance(self):
-        return get_storage_class(self.storage)(**self.settings)
+        return import_string(self.storage)(**self.settings)
 
     @property
     def settings(self):
         if not self.settings_data:
             return {}
-        return json.loads(self.settings_data)
+        try:
+            return json.loads(self.settings_data)
+        except Exception as e:
+            raise ValueError("Error in settings for storage '{0}' [{1}]".format(self.name, e))
+
+    def clean(self):
+        try:
+            self.test_storage()
+        except Exception as e:
+            raise ValidationError(str(e))
+
+    def test_storage(self):
+        # just do something that requires connection
+        try:
+            self.instance().listdir('')
+        except FileNotFoundError:
+            # FileSystemStorage doesn't create the base_dir until write
+            pass
+        return True
 
     def save(self, *args, **kwargs):
-        # check the settings are valid
+        # ensure the settings JSON isvalid
-        self.settings
+        if self.settings_data:
+            validate_json(self.settings_data)
+
         super(UserStorage, self).save(*args, **kwargs)
 
     def __str__(self):
         return self.name
+
+class BYOFieldFile(FieldFile):
+    
+    def __init__(self, instance, field, name):
+        logger.debug("BYOFieldFile(%r, %r, %r)", instance, field, name)
+        super().__init__(instance, field, name)
+        try:
+            self.storage = field.get_storage(instance)
+            logger.debug("Using BYO storage: %r", self.storage)
+        except ObjectDoesNotExist:
+            logger.debug("Unable to select BYO storage")
+            self.storage = None # trigger error if try to save etc
+
+class BYOStorageField(models.FileField):
+    attr_class = BYOFieldFile
+
+    def __init__(self, storage_instance, *args, **kwargs):
+        self.storage_instance = storage_instance
+        super().__init__(*args, **kwargs)
+
+    def deconstruct(self):
+        name, path, args, kwargs = super().deconstruct()
+        args = [self.storage_instance] + args
+        return name, path, args, kwargs
+
+    def get_storage(self, instance):
+
+        if callable(self.storage_instance):
+            storage = self.storage_instance(instance)
+        else:
+            storage = resolve_instance(instance, self.storage_instance)
+        
+        if not isinstance(storage, Storage):
+            raise RuntimeError("Not a storage instance")
+
+        return storage
+
+    #def formfield(self, **kwargs):
+    #    return BYOStorageFormField(**kwargs)
+
+

byostorage/multi.py

@@ -1,6 +1,7 @@
-from django.core.files.storage import Storage
+from django.core.files.storage import Storage, default_storage, storages
 from django.conf import settings
 from django.utils.deconstruct import deconstructible
+from django.utils.module_loading import import_string
 
 '''
 TODO: Create a signal to remove instances from cache if modified
@@ -24,8 +25,12 @@ class MultiStorage(Storage):
 
     def get_storage(self, name):
         if name not in self._cache:
-            config = self.config[name]
+            try:
-            self._cache[name] = get_storage_class(config['storage'])(**config['settings'])
+                config = self.config[name]
+            except KeyError:
+                return default_storage
+            klass = import_string(config.pop('storage', 'django.core.files.storage.FileSystemStorage'))
+            self._cache[name] = klass(**config)
         return self._cache[name]
 
     def split(self, name):
@@ -41,18 +46,21 @@ class MultiStorage(Storage):
     def _proxy(self, method, name, *args, **kwargs):
         storage, p = self.split(name)
         result =  getattr(self.get_storage(storage), method)(p, *args, **kwargs)
-        logger.debug("Proxy: %s(%r, *%r, **%r) => %r", method, name, args, kwargs, result)
+        logger.debug("Proxy: %s(%r, *%r, **%r) => %r", method, p, args, kwargs, result)
         return result
 
     def _proxy_name(self, method, name, *args, **kwargs):
         storage, p = self.split(name)
         result = getattr(self.get_storage(storage), method)(p, *args, **kwargs)
-        logger.debug("Proxy: %s(%r, *%r, **%r) => %s:%s", method, name, args, kwargs, storage, result)
+        logger.debug("Proxy: %s(%r, *%r, **%r) => %s:%s", method, p, args, kwargs, storage, result)
         return self.join(storage, result)
 
     def _open(self, name, mode='rb'):
         return self._proxy('open', name, mode)
 
+    def path(self, name):
+        return self._proxy('path', name)
+
     def save(self, name, content, max_length=None):
         return self._proxy_name('save', name, content, max_length)
 
@@ -77,21 +85,5 @@ class MultiStorage(Storage):
     def get_modified_time(self, name):
         return self._proxy('get_modified_time', name)
 
-
+    def __str__(self):
-class BYOStorage(MultiStorage):
+        return "<MultiStorage: {0} storages>".format(len(self.config))
-    ''' Database driven Bring-Your-Own-Storage
-
-    Multiple storages can 
-    '''
-
-    def get_storage(self, name):
-        if name not in self._cache:
-
-            # use storage from config by default
-            if name in self.config:
-                return super(BYOStorage, self).get_storage(name)
-
-            from .models import UserStorage
-            obj = UserStorage.objects.get(name=name)
-            self._cache[name] = obj.instance()
-        return self._cache[name]

byostorage/tests/test_byo_storage.py

@@ -0,0 +1,112 @@
+from django.test import TestCase
+from django.core.files.storage import FileSystemStorage
+from django.core.exceptions import ValidationError
+
+from byostorage.models import UserStorage
+from byostorage.user import BYOStorage
+
+#from example.models import LinkedStorageTestModel
+
+import tempfile
+import os.path
+
+#import logging
+#logging.basicConfig(level=logging.DEBUG)
+
+class UserStorageTestCase(TestCase):
+
+    def test_construction(self):
+        us = UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+                settings_data='{"location": "storage/one"}')
+
+        self.assertEqual(str(us), 'one')
+
+        storage = us.instance()
+        self.assertIsInstance(storage, FileSystemStorage)
+        self.assertEqual(storage.base_location, 'storage/one')
+
+    def test_bad_json(self):
+        with self.assertRaisesMessage(ValidationError, "Unterminated string"):
+            UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+            settings_data='{"location": "foo}')
+
+    def test_connection(self):
+        instance = UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
+                settings_data='''
+{
+  "access_key": "minioadmin",
+  "secret_key": "minioadmin",
+  "endpoint_url": "http://localhost:9000",
+  "bucket_name": "byostorage_test"
+}
+''')
+
+    
+    def test_bad_connection(self):
+        try:
+            UserStorage.objects.create(name='one', storage='storages.backends.s3boto3.S3Boto3Storage',
+                   settings_data='''
+{
+  "access_key": "minioadmin",
+  "secret_key": "the_wrong_secret",
+  "endpoint_url": "http://localhost:9000",
+  "bucket_name": "byostorage_test"
+}
+''')
+            self.fail("Should have raised an exception")
+        except Exception as e:
+            pass
+
+
+class BYOStorageTestCase(TestCase):
+
+    def setUp(self):
+        UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+                settings_data='{"location": "storage/one"}')
+    
+    def test_pickup_changes(self):
+        storage = BYOStorage()
+        one = storage.get_storage('one')
+        self.assertEqual(one.base_location, 'storage/one')
+
+        # if not already in cache then no issues
+        UserStorage.objects.create(name='two', storage='django.core.files.storage.FileSystemStorage',
+                settings_data='{"location": "storage/two"}')
+        two = storage.get_storage('two')
+        self.assertEqual(two.base_location, 'storage/two')
+
+        # modify the settings
+        o = UserStorage.objects.get(name='one')
+        o.settings_data = '{"location": "other/one"}'
+        o.save()
+
+        # signal should have fired and invalidated 'one'
+        one = storage.get_storage('one')
+        self.assertEqual(one.base_location, 'other/one')
+
+"""
+class BYOStorageFieldTestCase(TestCase):
+
+    def setUp(self):
+        self.test_dir = tempfile.TemporaryDirectory()
+        self.one = UserStorage.objects.create(name='one', storage='django.core.files.storage.FileSystemStorage',
+            settings_data = '{"location": "%s/one"}' % self.test_dir.name)
+
+    def tearDown(self):
+        self.test_dir.cleanup()
+
+    def test_linked_storage(self):
+        
+        from django.core.files.uploadedfile import UploadedFile
+
+        with open(__file__, 'rb') as f:
+            result = LinkedStorageTestModel.objects.create(name="first", my_storage_id=self.one.pk, my_file=UploadedFile(f))
+
+        expected = os.path.basename(__file__)
+
+        self.assertEqual(result.my_file.name, expected)
+        self.assertGreater(result.my_file.size, 0)
+
+        self.assertEqual(os.listdir(os.path.join(self.test_dir.name, 'one')), [expected])
+
+"""

byostorage/tests/test_http_storage.py

@@ -0,0 +1,29 @@
+from django.test import TestCase
+
+from byostorage.http import HTTPStorage
+
+class HTTPStorageTestCase(TestCase):
+
+    def test_url(self):
+        s = HTTPStorage()
+
+        self.assertEqual(s.url('//google.com'), 'https://google.com')
+
+    def test_exists(self):
+        s = HTTPStorage()
+
+        self.assertTrue(s.exists('//gitea.tfconsulting.com.au/tris'))
+        self.assertFalse(s.exists('//gitea.tfconsulting.com.au/foo.txt'))
+
+    def test_save(self):
+        return # already caught as suspicious activity
+        s = HTTPStorage()
+        with self.assertRaisesMessage(NotImplementedError, "Unable to save to web locations"):
+            s.save('//gitea.tfconsulting.com.au/foo', 'Some content')
+
+    def test_open(self):
+        s = HTTPStorage()
+        with s.open('//gitea.tfconsulting.com.au', 'rb') as f:
+            data = f.read()
+
+        self.assertTrue(len(data) > 4000)

byostorage/tests/test_multi_storage.py

@@ -0,0 +1,44 @@
+from django.test import TestCase
+from byostorage.multi import MultiStorage
+from django.core.files.storage import FileSystemStorage
+
+import os.path
+
+class MultiStorageTestCase(TestCase):
+
+    def test_storage_selection(self):
+        ms = MultiStorage({'one': {'location': 'storage/one'}, 'two': {}})
+        self.assertEqual(str(ms), "<MultiStorage: 2 storages>")
+
+        # use the default if nothing set
+        default = ms.get_storage('foo')
+        self.assertIsInstance(default, FileSystemStorage)
+        self.assertEqual(default.base_location, 'media')
+
+        # get first instance
+        one = ms.get_storage('one')
+        self.assertEqual(one.base_location, 'storage/one')
+
+    def test_storage_proxies(self):
+        ms = MultiStorage({
+            'one': {'location': 'storage/one', 'base_url': 'http://one'},
+            'two': {'location': 'storage/two', 'base_url': 'http://two'}
+        })
+
+
+        self.assertEqual(ms.url("one:foo.txt"), 'http://one/foo.txt')
+        self.assertEqual(ms.url("two:foo.txt"), 'http://two/foo.txt')
+
+    def test_with_http(self):
+        ms = MultiStorage({
+            'https': {'storage': 'byostorage.http.HTTPStorage', 'protocol': 'https'}
+        })
+
+        self.assertEqual(ms.url('https://google.com'), 'https://google.com')
+
+    def test_local_methods(self):
+        ms = MultiStorage({
+            'local': {'storage': 'django.core.files.storage.FileSystemStorage', 'location': 'local_store'}
+        })
+
+        self.assertEqual(ms.path('local:foo/bar.txt'), os.path.realpath('local_store/foo/bar.txt'))

byostorage/user.py

@@ -0,0 +1,32 @@
+from django.db.models.signals import post_save
+
+from .models import UserStorage
+from .multi import MultiStorage
+
+class BYOStorage(MultiStorage):
+    ''' Database driven Bring-Your-Own-Storage
+
+    '''
+
+    def __init__(self, config=None):
+        super(BYOStorage, self).__init__(config)
+
+        post_save.connect(self.handle_change, sender=UserStorage)
+
+
+    def handle_change(self, instance, **kwargs):
+        try:
+            del(self._cache[instance.name])
+        except KeyError:
+            pass
+
+    def get_storage(self, name):
+        if name not in self._cache:
+
+            # use storage from config by default
+            if name in self.config:
+                return super(BYOStorage, self).get_storage(name)
+
+            obj = UserStorage.objects.get(name=name)
+            self._cache[name] = obj.instance()
+        return self._cache[name]

setup.cfg

@@ -11,9 +11,8 @@ classifiers =
     Development Status :: 1 - Development
     Environment :: Web Environment
     Framework :: Django
-    Framework :: Django :: 2.2
+    Framework :: Django :: 4.2
-    Framework :: Django :: 3.0
+    Framework :: Django :: 5.0
-    Framework :: Django :: 3.1
     Intended Audience :: Developers
     License :: OSI Approved :: BSD License
     Operating System :: OS Independent
@@ -29,7 +28,7 @@ classifiers =
 [options]
 python_requires = >=3.5
 install_requires =
-    Django >= 2.2
+    Django >= 4.2
 packages =
     byostorage
     byostorage.migrations